source: rtems-source-builder/source-builder/sb/build.py @ 83586f7

4.104.114.95
Last change on this file since 83586f7 was 83586f7, checked in by Chris Johns <chrisj@…>, on 05/14/14 at 06:43:40

sb: Fix the formatting string.
  • Property mode set to 100644
File size: 18.3 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2013 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package given a config file. It only builds to be
22# installed not to be package unless you run a packager around this.
23#
24
25import getopt
26import glob
27import os
28import shutil
29import stat
30import sys
31import urllib2
32import urlparse
33
34try:
35    import check
36    import config
37    import download
38    import error
39    import ereport
40    import execute
41    import log
42    import options
43    import path
44    import sources
45    import version
46except KeyboardInterrupt:
47    print 'abort: user terminated'
48    sys.exit(1)
49except:
50    print 'error: unknown application load error'
51    sys.exit(1)
52
53class script:
54    """Create and manage a shell script."""
55
56    def __init__(self):
57        self.reset()
58
59    def reset(self):
60        self.body = []
61        self.lc = 0
62
63    def append(self, text):
64        if type(text) is str:
65            text = text.splitlines()
66        if not log.quiet:
67            i = 0
68            for l in text:
69                i += 1
70                log.output('script:%3d: %s' % (self.lc + i, l))
71        self.lc += len(text)
72        self.body.extend(text)
73
74    def write(self, name, check_for_errors = False):
75        s = None
76        try:
77            s = open(path.host(name), 'w')
78            s.write('\n'.join(self.body))
79            s.close()
80            os.chmod(path.host(name), stat.S_IRWXU | \
81                         stat.S_IRGRP | stat.S_IXGRP | \
82                         stat.S_IROTH | stat.S_IXOTH)
83        except IOError, err:
84            raise error.general('creating script: ' + name)
85        except:
86            if s is not None:
87                s.close()
88            raise
89        if s is not None:
90            s.close()
91
92class build:
93    """Build a package given a config file."""
94
95    def _name_(self, name):
96        #
97        # If on Windows use shorter names to keep the build paths.
98        #
99        if options.host_windows:
100            buildname = ''
101            add = True
102            for c in name:
103                if c == '-':
104                    add = True
105                elif add:
106                    buildname += c
107                    add = False
108            return buildname
109        else:
110            return name
111
112    def __init__(self, name, create_tar_files, opts, macros = None):
113        self.opts = opts
114        if macros is None:
115            self.macros = opts.defaults
116        else:
117            self.macros = macros
118        self.create_tar_files = create_tar_files
119        log.notice('config: ' + name)
120        self.config = config.file(name, opts, self.macros)
121        self.script = script()
122        self.macros['buildname'] = self._name_(self.macros['name'])
123
124    def rmdir(self, rmpath):
125        log.output('removing: %s' % (path.host(rmpath)))
126        if not self.opts.dry_run():
127            if path.exists(rmpath):
128                path.removeall(rmpath)
129
130    def mkdir(self, mkpath):
131        log.output('making dir: %s' % (path.host(mkpath)))
132        if not self.opts.dry_run():
133            path.mkdir(mkpath)
134
135    def canadian_cross(self):
136        _host = self.config.expand('%{_host}')
137        _build = self.config.expand('%{_build}')
138        _target = self.config.expand('%{_target}')
139        return self.config.defined('%{allow_cxc}') and \
140            _host != _build and _host != _target
141
142    def source(self, name):
143        #
144        # Return the list of sources. Merge in any macro defined sources as
145        # these may be overridden by user loaded macros.
146        #
147        _map = 'source-%s' % (name)
148        src_keys = self.macros.map_keys(_map)
149        if len(src_keys) == 0:
150            raise error.general('no source set: %s (%s)' % (name, _map))
151        srcs = []
152        for s in src_keys:
153            sm = self.macros.get(s, globals = False, maps = _map)
154            if sm is None:
155                raise error.internal('source macro not found: %s in %s (%s)' % \
156                                         (s, name, _map))
157            url = self.config.expand(sm[2])
158            src = download.parse_url(url, '_sourcedir', self.config, self.opts)
159            download.get_file(src['url'], src['local'], self.opts, self.config)
160            if 'symlink' in src:
161                src['script'] = '%%{__ln_s} %s ${source_dir_%s}' % (src['symlink'], name)
162            elif 'compressed' in src:
163                #
164                # Zip files unpack as well so do not use tar.
165                #
166                src['script'] = '%s %s' % (src['compressed'], src['local'])
167                if src['compressed-type'] != 'zip':
168                    src['script'] += ' | %{__tar_extract} -'
169            else:
170                src['script'] = '%%{__tar_extract} %s' % (src['local'])
171            srcs += [src]
172        return srcs
173
174    def source_setup(self, package, args):
175        log.output('source setup: %s: %s' % (package.name(), ' '.join(args)))
176        setup_name = args[1]
177        args = args[1:]
178        try:
179            opts, args = getopt.getopt(args[1:], 'qDcn:b:a:')
180        except getopt.GetoptError, ge:
181            raise error.general('source setup error: %s' % str(ge))
182        quiet = False
183        unpack_before_chdir = True
184        delete_before_unpack = True
185        create_dir = False
186        deleted_dir = False
187        created_dir = False
188        changed_dir = False
189        opt_name = None
190        for o in opts:
191            if o[0] == '-q':
192                quiet = True
193            elif o[0] == '-D':
194                delete_before_unpack = False
195            elif o[0] == '-c':
196                create_dir = True
197            elif o[0] == '-n':
198                opt_name = o[1]
199            elif o[0] == '-b':
200                unpack_before_chdir = True
201            elif o[0] == '-a':
202                unpack_before_chdir = False
203        name = None
204        for source in self.source(setup_name):
205            if name is None:
206                if opt_name is None:
207                    if source:
208                        opt_name = source['name']
209                    else:
210                        raise error.general('setup source tag not found: %d' % (source_tag))
211                else:
212                    name = opt_name
213                name = self._name_(name)
214            self.script.append(self.config.expand('cd %{_builddir}'))
215            if not deleted_dir and  delete_before_unpack:
216                self.script.append(self.config.expand('%{__rm} -rf ' + name))
217                deleted_dir = True
218            if not created_dir and create_dir:
219                self.script.append(self.config.expand('%{__mkdir_p} ' + name))
220                created_dir = True
221            if not changed_dir and (not unpack_before_chdir or create_dir):
222                self.script.append(self.config.expand('cd ' + name))
223                changed_dir = True
224            self.script.append(self.config.expand(source['script']))
225        if not changed_dir and (unpack_before_chdir and not create_dir):
226            self.script.append(self.config.expand('cd ' + name))
227            changed_dir = True
228        self.script.append(self.config.expand('%{__setup_post}'))
229
230    def patch_setup(self, package, args):
231        name = args[1]
232        args = args[2:]
233        _map = 'patch-%s' % (name)
234        default_opts = ' '.join(args)
235        patch_keys = self.macros.map_keys(_map)
236        patches = []
237        for p in patch_keys:
238            pm = self.macros.get(p, globals = False, maps = _map)
239            if pm is None:
240                raise error.internal('patch macro not found: %s in %s (%s)' % \
241                                         (p, name, _map))
242            opts = []
243            url = []
244            for pp in pm[2].split():
245                if len(url) == 0 and pp[0] == '-':
246                    opts += [pp]
247                else:
248                    url += [pp]
249            if len(url) == 0:
250                raise error.general('patch URL not found: %s' % (' '.join(args)))
251            if len(opts) == 0:
252                opts = default_opts
253            else:
254                opts = ' '.join(opts)
255            opts = self.config.expand(opts)
256            url = self.config.expand(' '.join(url))
257            #
258            # Parse the URL first in the source builder's patch directory.
259            #
260            patch = download.parse_url(url, '_patchdir', self.config, self.opts)
261            #
262            # If not in the source builder package check the source directory.
263            #
264            if not path.exists(patch['local']):
265                patch = download.parse_url(url, '_patchdir', self.config, self.opts)
266            download.get_file(patch['url'], patch['local'], self.opts, self.config)
267            if 'compressed' in patch:
268                patch['script'] = patch['compressed'] + ' ' +  patch['local']
269            else:
270                patch['script'] = '%{__cat} ' + patch['local']
271            patch['script'] += ' | %%{__patch} %s' % (opts)
272            self.script.append(self.config.expand(patch['script']))
273
274    def run(self, command, shell_opts = '', cwd = None):
275        e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
276        cmd = self.config.expand('%{___build_shell} -ex ' + shell_opts + ' ' + command)
277        log.output('run: ' + cmd)
278        exit_code, proc, output = e.shell(cmd, cwd = path.host(cwd))
279        if exit_code != 0:
280            log.output('shell cmd failed: %s' % (cmd))
281            raise error.general('building %s' % (self.macros['buildname']))
282
283    def builddir(self):
284        builddir = self.config.abspath('_builddir')
285        self.rmdir(builddir)
286        if not self.opts.dry_run():
287            self.mkdir(builddir)
288
289    def prep(self, package):
290        self.script.append('echo "==> %prep:"')
291        _prep = package.prep()
292        if _prep:
293            for l in _prep:
294                args = l.split()
295                if len(args):
296                    if args[0] == '%setup':
297                        if len(args) == 1:
298                            raise error.general('invalid %%setup directive: %s' % (' '.join(args)))
299                        if args[1] == 'source':
300                            self.source_setup(package, args[1:])
301                        elif args[1] == 'patch':
302                            self.patch_setup(package, args[1:])
303                    elif args[0].startswith('%patch'):
304                        self.patch(package, args)
305                    else:
306                        self.script.append(' '.join(args))
307
308    def build(self, package):
309        self.script.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"')
310        self.script.append('%s ${SB_BUILD_ROOT}' %
311                           (self.config.expand('%{__rmdir}')))
312        self.script.append('%s ${SB_BUILD_ROOT}' %
313                           (self.config.expand('%{__mkdir_p}')))
314        self.script.append('echo "==> %build:"')
315        _build = package.build()
316        if _build:
317            for l in _build:
318                self.script.append(l)
319
320    def install(self, package):
321        self.script.append('echo "==> %install:"')
322        _install = package.install()
323        if _install:
324            for l in _install:
325                args = l.split()
326                self.script.append(' '.join(args))
327
328    def files(self, package):
329        if self.create_tar_files \
330           and not self.macros.get('%{_disable_packaging'):
331            self.script.append('echo "==> %files:"')
332            inpath = path.abspath(self.config.expand('%{buildroot}'))
333            tardir = path.abspath(self.config.expand('%{_tardir}'))
334            self.script.append(self.config.expand('if test -d %s; then' % (inpath)))
335            self.script.append(self.config.expand('  %%{__mkdir_p} %s' % tardir))
336            self.script.append(self.config.expand('  cd ' + inpath))
337            tar = path.join(tardir, package.long_name() + '.tar.bz2')
338            cmd = self.config.expand('  %{__tar} -cf - . ' + '| %{__bzip2} > ' + tar)
339            self.script.append(cmd)
340            self.script.append(self.config.expand('  cd %{_builddir}'))
341            self.script.append('fi')
342
343    def clean(self, package):
344        self.script.append('echo "==> %clean:"')
345        _clean = package.clean()
346        if _clean is not None:
347            for l in _clean:
348                args = l.split()
349                self.script.append(' '.join(args))
350
351    def build_package(self, package):
352        if self.canadian_cross():
353            self.script.append('echo "==> Candian-cross build/target:"')
354            self.script.append('SB_CXC="yes"')
355        else:
356            self.script.append('SB_CXC="no"')
357        self.build(package)
358        self.install(package)
359        self.files(package)
360        if not self.opts.no_clean():
361            self.clean(package)
362
363    def cleanup(self):
364        package = self.main_package()
365        if not package.disabled() and not self.opts.no_clean():
366            buildroot = self.config.abspath('buildroot')
367            builddir = self.config.abspath('_builddir')
368            buildcxcdir = self.config.abspath('_buildcxcdir')
369            tmproot = self.config.abspath('_tmproot')
370            log.trace('cleanup: %s' % (buildroot))
371            self.rmdir(buildroot)
372            log.trace('cleanup: %s' % (builddir))
373            self.rmdir(builddir)
374            if self.canadian_cross():
375                log.trace('cleanup: %s' % (buildcxcdir))
376                self.rmdir(buildcxcdir)
377            log.trace('cleanup: %s' % (tmproot))
378            self.rmdir(tmproot)
379
380    def main_package(self):
381        packages = self.config.packages()
382        return packages['main']
383
384    def make(self):
385        package = self.main_package()
386        if package.disabled():
387            log.notice('package: nothing to build')
388        else:
389            name = package.name()
390            if self.canadian_cross():
391                log.notice('package: (Cxc) %s' % (name))
392            else:
393                log.notice('package: %s' % (name))
394                log.trace('---- macro maps %s' % ('-' * 55))
395                log.trace('%s' % (str(self.config.macros)))
396                log.trace('-' * 70)
397            self.script.reset()
398            self.script.append(self.config.expand('%{___build_template}'))
399            self.script.append('echo "=> ' + name + ':"')
400            self.prep(package)
401            self.build_package(package)
402            if not self.opts.dry_run():
403                self.builddir()
404                sn = path.join(self.config.expand('%{_builddir}'), 'doit')
405                log.output('write script: ' + sn)
406                self.script.write(sn)
407                if self.canadian_cross():
408                    log.notice('building: (Cxc) %s' % (name))
409                else:
410                    log.notice('building: %s' % (name))
411                self.run(sn)
412
413    def name(self):
414        packages = self.config.packages()
415        package = packages['main']
416        return package.name()
417
418    def disabled(self):
419        packages = self.config.packages()
420        package = packages['main']
421        return package.disabled()
422
423def get_configs(opts):
424
425    def _scan(_path, ext):
426        configs = []
427        for root, dirs, files in os.walk(_path):
428            prefix = root[len(_path) + 1:]
429            for file in files:
430                for e in ext:
431                    if file.endswith(e):
432                        configs += [path.join(prefix, file)]
433        return configs
434
435    configs = { 'paths': [], 'files': [] }
436    for cp in opts.defaults.expand('%{_configdir}').split(':'):
437        hcp = path.host(path.abspath(cp))
438        configs['paths'] += [hcp]
439        configs['files'] += _scan(hcp, ['.cfg', '.bset'])
440    configs['files'] = sorted(configs['files'])
441    return configs
442
443def find_config(config, configs):
444    config_root, config_ext = path.splitext(config)
445    if config_ext not in ['', '.bset', '.cfg']:
446        config_root = config
447        config_ext = ''
448    for c in configs['files']:
449        r, e = path.splitext(c)
450        if config_root == r:
451            if config_ext == '' or config_ext == e:
452                return c
453    return None
454
455def run(args):
456    ec = 0
457    opts = None
458    b = None
459    erheader = None
460    try:
461        optargs = { '--list-configs': 'List available configurations' }
462        opts = options.load(args, optargs)
463        log.notice('RTEMS Source Builder, Package Builder v%s' % (version.str()))
464        if not check.host_setup(opts):
465            if not opts.force():
466                raise error.general('host build environment is not set up' +
467                                    ' correctly (use --force to proceed)')
468            log.notice('warning: forcing build with known host setup problems')
469        if opts.get_arg('--list-configs'):
470            configs = get_configs(opts)
471            for p in configs['paths']:
472                print 'Examining: %s' % (os.path.relpath(p))
473                for c in configs['files']:
474                    if c.endswith('.cfg'):
475                        print '    %s' % (c)
476        else:
477            for config_file in opts.config_files():
478                b = build(config_file, True, opts)
479                b.make()
480                b = None
481    except error.general, gerr:
482        erheader = 'Build: %s' % (gerr)
483        log.notice(str(gerr))
484        log.stderr('Build FAILED')
485        ec = 1
486    except error.internal, ierr:
487        erheader = 'Build: %s' % (ierr)
488        log.notice(str(ierr))
489        log.stderr('Internal Build FAILED')
490        ec = 1
491    except error.exit, eerr:
492        pass
493    except KeyboardInterrupt:
494        log.notice('abort: user terminated')
495        ec = 1
496    if (ec != 0 and erheader and opts and b) or (opts and opts.dry_run()):
497        if opts.dry_run():
498            bname = 'dry-run'
499        else:
500            bname = b.name()
501        ereport.generate('rsb-report-%s.txt' % (bname), opts, erheader)
502    sys.exit(ec)
503
504if __name__ == "__main__":
505    run(sys.argv)
Note: See TracBrowser for help on using the repository browser.