source: rtems-source-builder/source-builder/sb/setbuilder.py @ 0565e1f

4.104.114.95
Last change on this file since 0565e1f was cb12e48, checked in by Chris Johns <chrisj@…>, on 04/09/13 at 03:51:43

Refactor defaults, macros and options.

To support building snapshots and pre-release source the defaults
has been refactored. The defaults have been moved to a stand alone
file and a macros.py module added. This modile abstracts the
old default dictionary turning it into a class. The macros
class can load macros from a file therefore the defaults have
been moved to a stand alone file.

The use of defaults has been removed from the project. The only
case where it is used in the options where the defaults are read
from a file. Macros are used everywhere now.

The defaults.py has been moved to the option.py and the separate
options and defaults values has been moved to a new pattern. When
constructing an object that needs macros and options if the macros
passed in is None the defaults from the options are used. This makes
it clear when the defaults are being used or when a modified set of
macros is being used.

The macros class support maps. The default is 'global' and where all
the defaults reside and where configuratiion file changes end up.
Maps allow macros to be read from a file and override the values
being maintained in the 'global' map. Reading a macro first checks
the map and if not present checks the 'global' map.

The addition of maps to the macros provides the base to support
snapshots and pre-release testing with standard configurations.
This functionality needs to be added. It works by letting to
specify a snapshot with:

source0: none, override, 'my-dist.tar.bz2'

and it will be used rather the value from the standard configuration.
With a build set you need to also specify the package these macros
are for. The maps provide this.
  • Property mode set to 100644
File size: 14.4 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2013 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package compiler tool suite given a tool set. A tool
22# set lists the various tools. These are specific tool configurations.
23#
24
25import copy
26import datetime
27import distutils.dir_util
28import glob
29import operator
30import os
31import sys
32
33try:
34    import build
35    import check
36    import error
37    import log
38    import options
39    import path
40    import reports
41    import version
42except KeyboardInterrupt:
43    print 'abort: user terminated'
44    sys.exit(1)
45except:
46    print 'error: unknown application load error'
47    sys.exit(1)
48
49def _trace(opts, text):
50    if opts.trace():
51        print text
52
53def _notice(opts, text):
54    if not opts.quiet() and not log.default.has_stdout():
55        print text
56    log.output(text)
57    log.flush()
58
59class buildset:
60    """Build a set builds a set of packages."""
61
62    def __init__(self, bset, _configs, opts, macros = None):
63        _trace(opts, '_bset:%s: init' % (bset))
64        self.configs = _configs
65        self.opts = opts
66        if macros is None:
67            self.macros = copy.copy(opts.defaults)
68        else:
69            self.macros = copy.copy(macros)
70        self.bset = bset
71        self.bset_pkg = '%s-%s-set' % (self.macros.expand('%{_target}'), self.bset)
72
73    def _output(self, text):
74        if not self.opts.quiet():
75            log.output(text)
76
77    def copy(self, src, dst):
78        if not os.path.isdir(path.host(src)):
79            raise error.general('copying tree: no source directory: %s' % (path.host(src)))
80        if not self.opts.dry_run():
81            try:
82                files = distutils.dir_util.copy_tree(path.host(src),
83                                                     path.host(dst))
84                for f in files:
85                    self._output(f)
86            except IOError, err:
87                raise error.general('copying tree: %s: %s' % (what, str(err)))
88            except distutils.errors.DistutilsFileError, err:
89                raise error.general('copying tree: %s' % (str(err)))
90
91    def report(self, _config, _build):
92        if not _build.opts.get_arg('--no-report'):
93            format = _build.opts.get_arg('--report-format')
94            if format is None:
95                format = 'html'
96                ext = '.html'
97            else:
98                if len(format) != 2:
99                    raise error.general('invalid report format option: %s' % ('='.join(format)))
100                if format[1] == 'text':
101                    format = 'text'
102                    ext = '.txt'
103                elif format[1] == 'asciidoc':
104                    format = 'asciidoc'
105                    ext = '.txt'
106                elif format[1] == 'html':
107                    format = 'html'
108                    ext = '.html'
109                else:
110                    raise error.general('invalid report format: %s' % (format[1]))
111            buildroot = _build.config.abspath('%{buildroot}')
112            prefix = _build.macros.expand('%{_prefix}')
113            name = _build.main_package().name() + ext
114            outpath = path.host(path.join(buildroot, prefix, 'share', 'rtems-source-builder'))
115            outname = path.host(path.join(outpath, name))
116            _notice(self.opts, 'reporting: %s -> %s' % (_config, name))
117            if not _build.opts.dry_run():
118                _build.mkdir(outpath)
119                r = reports.report(format, self.configs, _build.opts, _build.macros)
120                r.make(_config, outname)
121                del r
122
123    def root_copy(self, src, dst):
124        what = '%s -> %s' % \
125            (os.path.relpath(path.host(src)), os.path.relpath(path.host(dst)))
126        if self.opts.trace():
127            _notice(self.opts, 'collecting: %s' % (what))
128        if not self.opts.dry_run():
129            self.copy(src, dst)
130
131    def install(self, name, buildroot, prefix):
132        dst = prefix
133        src = path.join(buildroot, prefix)
134        _notice(self.opts, 'installing: %s -> %s' % (name, path.host(dst)))
135        if not self.opts.dry_run():
136            self.copy(src, dst)
137
138    def canadian_cross(self, _build):
139        # @fixme Switch to using a private macros map.
140        macros_to_save = ['%{_prefix}',
141                          '%{_tmproot}',
142                          '%{buildroot}',
143                          '%{_builddir}',
144                          '%{_host}']
145        macros_to_copy = [('%{_host}',     '%{_build}'),
146                          ('%{_tmproot}',  '%{_tmpcxcroot}'),
147                          ('%{buildroot}', '%{buildcxcroot}'),
148                          ('%{_builddir}', '%{_buildcxcdir}')]
149        orig_macros = {}
150        for m in macros_to_save:
151            orig_macros[m] = _build.config.macro(m)
152        for m in macros_to_copy:
153            _build.config.set_define(m[0], _build.config.macro(m[1]))
154        _build.make()
155        for m in macris_to_save:
156            _build.config.set_define(m, orig_macros[m])
157        self.root_copy(_build.config.expand('%{buildcxcroot}'),
158                       _build.config.expand('%{_tmpcxcroot}'))
159
160    def build_package(self, _config, _build):
161        if _build.canadian_cross():
162            self.canadian_cross(_build)
163        _build.make()
164        self.report(_config, _build)
165        self.root_copy(_build.config.expand('%{buildroot}'),
166                       _build.config.expand('%{_tmproot}'))
167
168    def bset_tar(self, _build):
169        tardir = _build.config.expand('%{_tardir}')
170        if self.opts.get_arg('--bset-tar-file'):
171            path.mkdir(tardir)
172            tar = path.join(tardir, _build.config.expand('%s.tar.bz2' % (self.bset_pkg)))
173            _notice(self.opts, 'tarball: %s' % (os.path.relpath(path.host(tar))))
174            if not self.opts.dry_run():
175                tmproot = _build.config.expand('%{_tmproot}')
176                cmd = _build.config.expand("'cd " + tmproot + \
177                                               " && %{__tar} -cf - . | %{__bzip2} > " + tar + "'")
178                _build.run(cmd, shell_opts = '-c', cwd = tmproot)
179
180    def parse(self, bset):
181
182        def _clean(line):
183            line = line[0:-1]
184            b = line.find('#')
185            if b >= 0:
186                line = line[1:b]
187            return line.strip()
188
189        bsetname = bset
190
191        if not path.exists(bsetname):
192            for cp in self.macros.expand('%{_configdir}').split(':'):
193                configdir = path.abspath(cp)
194                bsetname = path.join(configdir, bset)
195                if path.exists(bsetname):
196                    break
197                bsetname = None
198            if bsetname is None:
199                raise error.general('no build set file found: %s' % (bset))
200        try:
201            if self.opts.trace():
202                print '_bset:%s: open: %s' % (self.bset, bsetname)
203            bset = open(path.host(bsetname), 'r')
204        except IOError, err:
205            raise error.general('error opening bset file: %s' % (bsetname))
206
207        configs = []
208
209        try:
210            lc = 0
211            for l in bset:
212                lc += 1
213                l = _clean(l)
214                if len(l) == 0:
215                    continue
216                if self.opts.trace():
217                    print '%03d: %s' % (lc, l)
218                ls = l.split()
219                if ls[0][-1] == ':' and ls[0][:-1] == 'package':
220                    self.bset_pkg = self.macros.expand(ls[1].strip())
221                    self.macros['package'] = self.bset_pkg
222                elif ls[0][0] == '%':
223                    if ls[0] == '%define':
224                        if len(ls) > 2:
225                            self.macros.define(ls[1].strip(),
226                                               ' '.join([f.strip() for f in ls[2:]]))
227                        else:
228                            self.macros.define(ls[1].strip())
229                    elif ls[0] == '%undefine':
230                        if len(ls) > 2:
231                            raise error.general('%undefine requires just the name')
232                        self.macros.undefine(ls[1].strip())
233                    elif ls[0] == '%include':
234                        configs += self.parse(ls[1].strip())
235                    else:
236                        raise error.general('invalid directive in build set files: %s' % (l))
237                else:
238                    l = l.strip()
239                    c = build.find_config(l, self.configs)
240                    if c is None:
241                        raise error.general('cannot find file: %s' % (l))
242                    configs += [c]
243        except:
244            bset.close()
245            raise
246
247        bset.close()
248
249        return configs
250
251    def load(self):
252
253        exbset = self.macros.expand(self.bset)
254
255        self.macros['_bset'] = exbset
256
257        root, ext = path.splitext(exbset)
258
259        if exbset.endswith('.bset'):
260            bset = exbset
261        else:
262            bset = '%s.bset' % (exbset)
263
264        return self.parse(bset)
265
266    def build(self, deps = None):
267
268        _trace(self.opts, '_bset:%s: make' % (self.bset))
269        _notice(self.opts, 'Build Set: %s' % (self.bset))
270
271        configs = self.load()
272
273        _trace(self.opts, '_bset:%s: configs: %s'  % (self.bset, ','.join(configs)))
274
275        current_path = os.environ['PATH']
276
277        start = datetime.datetime.now()
278
279        try:
280            builds = []
281            for s in range(0, len(configs)):
282                try:
283                    #
284                    # Each section of the build set gets a separate set of
285                    # macros so we do not contaminate one configuration with
286                    # another.
287                    #
288                    opts = copy.copy(self.opts)
289                    macros = copy.copy(self.macros)
290                    if configs[s].endswith('.bset'):
291                        bs = buildset(configs[s], self.configs, opts, macros)
292                        bs.build(deps)
293                        del bs
294                    elif configs[s].endswith('.cfg'):
295                        b = build.build(configs[s], self.opts.get_arg('--pkg-tar-files'),
296                                        opts, macros)
297                        if deps is None:
298                            self.build_package(configs[s], b)
299                            if s == len(configs) - 1:
300                                self.bset_tar(b)
301                        else:
302                            deps += b.config.includes()
303                        builds += [b]
304                    else:
305                        raise error.general('invalid config type: %s' % (configs[s]))
306                except error.general, gerr:
307                    if self.opts.keep_going():
308                        print gerr
309                        if self.opts.always_clean():
310                            builds += [b]
311                    else:
312                        raise
313            if deps is None and not self.opts.get_arg('--no-install'):
314                for b in builds:
315                    self.install(b.name(),
316                                 b.config.expand('%{buildroot}'),
317                                 b.config.expand('%{_prefix}'))
318            if deps is None and \
319                    (not self.opts.no_clean() or self.opts.always_clean()):
320                for b in builds:
321                    _notice(self.opts, 'cleaning: %s' % (b.name()))
322                    b.cleanup()
323            for b in builds:
324                del b
325        except:
326            os.environ['PATH'] = current_path
327            raise
328
329        end = datetime.datetime.now()
330
331        os.environ['PATH'] = current_path
332
333        _notice(self.opts, 'Build Set: Time %s' % (str(end - start)))
334
335def list_bset_cfg_files(opts, configs):
336    if opts.get_arg('--list-configs') or opts.get_arg('--list-bsets'):
337        if opts.get_arg('--list-configs'):
338            ext = '.cfg'
339        else:
340            ext = '.bset'
341        for p in configs['paths']:
342            print 'Examining: %s' % (os.path.relpath(p))
343        for c in configs['files']:
344            if c.endswith(ext):
345                print '    %s' % (c)
346        return True
347    return False
348
349def run():
350    import sys
351    try:
352        optargs = { '--list-configs':  'List available configurations',
353                    '--list-bsets':    'List available build sets',
354                    '--list-deps':     'List the dependent files.',
355                    '--no-install':    'Do not install the packages to the prefix.',
356                    '--no-report':     'Do not create a package report.',
357                    '--report-format': 'The report format (text, html, asciidoc).',
358                    '--bset-tar-file': 'Create a build set tar file',
359                    '--pkg-tar-files': 'Create package tar files' }
360        opts = options.load(sys.argv, optargs)
361        log.default = log.log(opts.logfiles())
362        _notice(opts, 'RTEMS Source Builder - Set Builder, v%s' % (version.str()))
363        if not check.host_setup(opts):
364            raise error.general('host build environment is not set up correctly')
365        configs = build.get_configs(opts)
366        if opts.get_arg('--list-deps'):
367            deps = []
368        else:
369            deps = None
370        if not list_bset_cfg_files(opts, configs):
371            for bset in opts.params():
372                b = buildset(bset, configs, opts)
373                b.build(deps)
374                del b
375        if deps is not None:
376            c = 0
377            for d in sorted(set(deps)):
378                c += 1
379                print 'dep[%d]: %s' % (c, d)
380    except error.general, gerr:
381        print gerr
382        print >> sys.stderr, 'Build FAILED'
383        sys.exit(1)
384    except error.internal, ierr:
385        print ierr
386        sys.exit(1)
387    except error.exit, eerr:
388        pass
389    except KeyboardInterrupt:
390        _notice(opts, 'abort: user terminated')
391        sys.exit(1)
392    sys.exit(0)
393
394if __name__ == "__main__":
395    run()
Note: See TracBrowser for help on using the repository browser.