source: rtems-source-builder/sb/build.py @ ec26725

4.104.114.95
Last change on this file since ec26725 was ec26725, checked in by Chris Johns <chrisj@…>, on 11/05/12 at 12:12:01

Add support for atch paths.
  • Property mode set to 100644
File size: 17.3 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2012 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package given a config file. It only builds to be
22# installed not to be package unless you run a packager around this.
23#
24
25import getopt
26import glob
27import os
28import shutil
29import stat
30import sys
31import urllib2
32import urlparse
33
34import check
35import config
36import defaults
37import error
38import execute
39import log
40import path
41
42#
43# Version of Sourcer Builder Build.
44#
45version = '0.1'
46
47def _notice(opts, text):
48    if not opts.quiet() and not log.default.has_stdout():
49        print text
50    log.output(text)
51    log.flush()
52
53class script:
54    """Create and manage a shell script."""
55
56    def __init__(self, quiet = True):
57        self.quiet = quiet
58        self.reset()
59
60    def reset(self):
61        self.body = []
62        self.lc = 0
63
64    def append(self, text):
65        if type(text) is str:
66            text = text.splitlines()
67        if not self.quiet:
68            i = 0
69            for l in text:
70                i += 1
71                log.output('script:%3d: ' % (self.lc + i) + l)
72        self.lc += len(text)
73        self.body.extend(text)
74
75    def write(self, name, check_for_errors = False):
76        s = None
77        try:
78            s = open(path.host(name), 'w')
79            s.write('\n'.join(self.body))
80            s.close()
81            os.chmod(path.host(name), stat.S_IRWXU | \
82                         stat.S_IRGRP | stat.S_IXGRP | \
83                         stat.S_IROTH | stat.S_IXOTH)
84        except IOError, err:
85            raise error.general('creating script: ' + name)
86        except:
87            if s is not None:
88                s.close()
89            raise
90        if s is not None:
91            s.close()
92
93class build:
94    """Build a package given a config file."""
95
96    def __init__(self, name, _defaults, opts):
97        self.opts = opts
98        _notice(opts, 'building: ' + name)
99        self.config = config.file(name, _defaults = _defaults, opts = opts)
100        self.script = script(quiet = opts.quiet())
101
102    def _output(self, text):
103        if not self.opts.quiet():
104            log.output(text)
105
106    def rmdir(self, rmpath):
107        self._output('removing: %s' % (path.host(rmpath)))
108        if not self.opts.dry_run():
109            if path.exists(rmpath):
110                try:
111                    shutil.rmtree(path.host(rmpath))
112                except IOError, err:
113                    raise error.error('error removing: %s' % (rmpath))
114                except WindowsError, err:
115                    _notice(self.opts, 'warning: cannot remove: %s' % (rmpath))
116
117    def mkdir(self, mkpath):
118        self._output('making dir: %s' % (path.host(mkpath)))
119        if not self.opts.dry_run():
120            try:
121                os.makedirs(path.host(mkpath))
122            except IOError, err:
123                _notice(self.opts, 'warning: cannot make directory: %s' % (mkpath))
124            except WindowsError, err:
125                _notice(self.opts, 'warning: cannot make directory: %s' % (mkpath))
126
127    def get_file(self, url, local):
128        if local is None:
129            raise error.general('source/patch path invalid')
130        if not path.isdir(path.dirname(local)):
131            if not self.opts.force():
132                raise error.general('source path not found: %s; (--force to create)' \
133                                        % (path.host(path.dirname(local))))
134            self.mkdir(path.host(path.dirname(local)))
135        if not path.exists(local):
136            #
137            # Not localy found so we need to download it. Check if a URL has
138            # been provided on the command line.
139            #
140            url_bases = self.opts.urls()
141            urls = []
142            if url_bases is not None:
143                for base in url_bases:
144                    if base[-1:] != '/':
145                        base += '/'
146                    url_path = urlparse.urlsplit(url)[2]
147                    slash = url_path.rfind('/')
148                    if slash < 0:
149                        url_file = url_path
150                    else:
151                        url_file = url_path[slash + 1:]
152                    urls.append(urlparse.urljoin(base, url_file))
153            urls.append(url)
154            if self.opts.trace():
155                print '_url:', ','.join(urls), '->', local
156            for url in urls:
157                #
158                # Hack for GitHub.
159                #
160                if url.startswith('https://api.github.com'):
161                    url = urlparse.urljoin(url, self.config.expand('tarball/%{version}'))
162                _notice(self.opts, 'download: %s -> %s' % (url, path.host(local)))
163                if not self.opts.dry_run():
164                    failed = False
165                    _in = None
166                    _out = None
167                    try:
168                        _in = urllib2.urlopen(url)
169                        _out = open(path.host(local), 'wb')
170                        _out.write(_in.read())
171                    except IOError, err:
172                        msg = 'download: %s: error: %s' % (url, str(err))
173                        _notice(self.opts, msg)
174                        if path.exists(local):
175                            os.remove(path.host(local))
176                        failed = True
177                    except:
178                        print >> sys.stderr, msg
179                        if _out is not None:
180                            _out.close()
181                        raise
182                    if _out is not None:
183                        _out.close()
184                    if _in is not None:
185                        del _in
186                    if not failed:
187                        if not path.isfile(local):
188                            raise error.general('source is not a file: %s' % (path.host(local)))
189                        return
190            if not self.opts.dry_run():
191                raise error.general('downloading %s: all paths have failed, giving up' % (url))
192
193    def parse_url(self, url, pathkey):
194        #
195        # Split the source up into the parts we need.
196        #
197        source = {}
198        source['url'] = url
199        source['path'] = path.dirname(url)
200        source['file'] = path.basename(url)
201        source['name'], source['ext'] = path.splitext(source['file'])
202        #
203        # Get the file. Checks the local source directory first.
204        #
205        source['local'] = None
206        for p in self.config.define(pathkey).split(':'):
207            local = path.join(path.abspath(p), source['file'])
208            if source['local'] is None or path.exists(local):
209                source['local'] = local
210                break
211        #
212        # Is the file compressed ?
213        #
214        esl = source['ext'].split('.')
215        if esl[-1:][0] == 'gz':
216            source['compressed'] = '%{__gzip} -dc'
217        elif esl[-1:][0] == 'bz2':
218            source['compressed'] = '%{__bzip2} -dc'
219        elif esl[-1:][0] == 'bz2':
220            source['compressed'] = '%{__zip} -u'
221        elif esl[-1:][0] == 'xz':
222            source['compressed'] = '%{__xz} -dc'
223        source['script'] = ''
224        return source
225
226    def source(self, package, source_tag):
227        #
228        # Scan the sources found in the config file for the one we are
229        # after. Infos or tags are lists.
230        #
231        sources = package.sources()
232        url = None
233        for s in sources:
234            tag = s[len('source'):]
235            if tag.isdigit():
236                if int(tag) == source_tag:
237                    url = sources[s][0]
238                    break
239        if url is None:
240            raise error.general('source tag not found: source%d' % (source_tag))
241        source = self.parse_url(url, '_sourcedir')
242        self.get_file(source['url'], source['local'])
243        if 'compressed' in source:
244            source['script'] = source['compressed'] + ' ' + \
245                source['local'] + ' | %{__tar_extract} -'
246        else:
247            source['script'] = '%{__tar_extract} ' + source['local']
248        return source
249
250    def patch(self, package, args):
251        #
252        # Scan the patches found in the config file for the one we are
253        # after. Infos or tags are lists.
254        #
255        patches = package.patches()
256        url = None
257        for p in patches:
258            if args[0][1:].lower() == p:
259                url = patches[p][0]
260                break
261        if url is None:
262            raise error.general('patch tag not found: %s' % (args[0]))
263        #
264        # Parse the URL first in the source builder's patch directory.
265        #
266        patch = self.parse_url(url, '_patchdir')
267        #
268        # If not in the source builder package check the source directory.
269        #
270        if not path.exists(patch['local']):
271            patch = self.parse_url(url, '_sourcedir')
272        self.get_file(patch['url'], patch['local'])
273        if 'compressed' in patch:
274            patch['script'] = patch['compressed'] + ' ' +  patch['local']
275        else:
276            patch['script'] = '%{__cat} ' + patch['local']
277        patch['script'] += ' | %{__patch} ' + ' '.join(args[1:])
278        self.script.append(self.config.expand(patch['script']))
279
280    def setup(self, package, args):
281        self._output('prep: %s: %s' % (package.name(), ' '.join(args)))
282        opts, args = getopt.getopt(args[1:], 'qDcTn:b:a:')
283        source_tag = 0
284        quiet = False
285        unpack_default_source = True
286        delete_before_unpack = True
287        create_dir = False
288        name = None
289        unpack_before_chdir = True
290        for o in opts:
291            if o[0] == '-q':
292                quiet = True
293            elif o[0] == '-D':
294                delete_before_unpack = False
295            elif o[0] == '-c':
296                create_dir = True
297            elif o[0] == '-T':
298                unpack_default_source = False
299            elif o[0] == '-n':
300                name = o[1]
301            elif o[0] == '-b':
302                unpack_before_chdir = True
303                if not o[1].isdigit():
304                    raise error.general('setup source tag no a number: %s' % (o[1]))
305                source_tag = int(o[1])
306            elif o[0] == '-a':
307                unpack_before_chdir = False
308                source_tag = int(o[1])
309        source0 = None
310        source = self.source(package, source_tag)
311        if name is None:
312            if source:
313                name = source['name']
314            else:
315                name = source0['name']
316        self.script.append(self.config.expand('cd %{_builddir}'))
317        if delete_before_unpack:
318            self.script.append(self.config.expand('%{__rm} -rf ' + name))
319        if create_dir:
320            self.script.append(self.config.expand('%{__mkdir_p} ' + name))
321        #
322        # If -a? then change directory before unpacking.
323        #
324        if not unpack_before_chdir:
325            self.script.append(self.config.expand('cd ' + name))
326        #
327        # Unpacking the source. Note, treated the same as -a0.
328        #
329        if unpack_default_source and source_tag != 0:
330            source0 = self.source(package, 0)
331            if source0 is None:
332                raise error.general('no setup source0 tag found')
333            self.script.append(self.config.expand(source0['script']))
334        self.script.append(self.config.expand(source['script']))
335        if unpack_before_chdir:
336            self.script.append(self.config.expand('cd ' + name))
337        self.script.append(self.config.expand('%{__setup_post}'))
338        if create_dir:
339            self.script.append(self.config.expand('cd ..'))
340
341    def run(self, command, shell_opts = '', cwd = None):
342        e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
343        cmd = self.config.expand('%{___build_shell} -ex ' + shell_opts + ' ' + command)
344        self._output('run: ' + cmd)
345        exit_code, proc, output = e.shell(cmd, cwd = path.host(cwd))
346        if exit_code != 0:
347            raise error.general('shell cmd failed: ' + cmd)
348
349    def builddir(self):
350        builddir = self.config.abspath('_builddir')
351        self.rmdir(builddir)
352        if not self.opts.dry_run():
353            self.mkdir(builddir)
354
355    def prep(self, package):
356        self.script.append('echo "==> %prep:"')
357        _prep = package.prep()
358        for l in _prep:
359            args = l.split()
360            if args[0] == '%setup':
361                self.setup(package, args)
362            elif args[0].startswith('%patch'):
363                self.patch(package, args)
364            else:
365                self.script.append(' '.join(args))
366
367    def build(self, package):
368        self.script.append('echo "==> %build:"')
369        _build = package.build()
370        for l in _build:
371            args = l.split()
372            self.script.append(' '.join(args))
373
374    def install(self, package):
375        self.script.append('echo "==> %install:"')
376        _install = package.install()
377        for l in _install:
378            args = l.split()
379            self.script.append(' '.join(args))
380
381    def files(self, package):
382        self.script.append('echo "==> %files:"')
383        prefixbase = self.opts.prefixbase()
384        if prefixbase is None:
385            prefixbase = ''
386        inpath = path.join('%{buildroot}', prefixbase)
387        tardir = path.abspath(self.config.expand('%{_tardir}'))
388        self.script.append(self.config.expand('if test -d %s; then' % (inpath)))
389        self.script.append('  mkdir -p %s' % tardir)
390        self.script.append(self.config.expand('  cd ' + inpath))
391        tar = path.join(tardir, package.long_name() + '.tar.bz2')
392        cmd = self.config.expand('  %{__tar} -cf - . ' + '| %{__bzip2} > ' + tar)
393        self.script.append(cmd)
394        self.script.append(self.config.expand('  cd %{_builddir}'))
395        self.script.append('fi')
396
397    def clean(self, package):
398        self.script.append('echo "==> %clean:"')
399        _clean = package.clean()
400        if _clean is not None:
401            for l in _clean:
402                args = l.split()
403                self.script.append(' '.join(args))
404
405    def cleanup(self):
406        if not self.opts.no_clean():
407            buildroot = self.config.abspath('buildroot')
408            builddir = self.config.abspath('_builddir')
409            _notice(self.opts, 'cleanup: %s' % (buildroot))
410            self.rmdir(buildroot)
411            _notice(self.opts, 'cleanup: %s' % (builddir))
412            self.rmdir(builddir)
413
414    def make(self):
415        packages = self.config.packages()
416        package = packages['main']
417        name = package.name()
418        _notice(self.opts, 'package: %s' % (name))
419        self.script.reset()
420        self.script.append(self.config.expand('%{___build_template}'))
421        self.script.append('echo "=> ' + name + ':"')
422        self.prep(package)
423        self.build(package)
424        self.install(package)
425        self.files(package)
426        if not self.opts.no_clean():
427            self.clean(package)
428        if not self.opts.dry_run():
429            self.builddir()
430            sn = path.join(self.config.expand('%{_builddir}'), 'doit')
431            self._output('write script: ' + sn)
432            self.script.write(sn)
433            _notice(self.opts, 'building: ' + name)
434            self.run(sn)
435
436    def name(self):
437        packages = self.config.packages()
438        package = packages['main']
439        return package.name()
440
441def list_configs(opts, _defaults, ext = '.cfg'):
442    configs = []
443    for cp in opts.expand('%{_configdir}', _defaults).split(':'):
444        print 'Examining: %s' % (path.host(path.abspath(cp)))
445        configs += glob.glob(path.host(path.join(cp, '*%s' % (ext))))
446    for c in sorted(configs):
447        config = path.basename(c)
448        if config.endswith(ext):
449            config = config[:0 - len(ext)]
450        print ' ', config
451
452def run(args):
453    try:
454        optargs = { '--list-configs': 'List available configurations' }
455        opts, _defaults = defaults.load(args, optargs)
456        log.default = log.log(opts.logfiles())
457        _notice(opts, 'Source Builder, Package Builder v%s' % (version))
458        if not check.host_setup(opts, _defaults):
459            if not opts.force():
460                raise error.general('host build environment is not set up correctly (use --force to proceed)')
461            _notice(opts, 'warning: forcing build with known host setup problems')
462        if opts.get_arg('--list-configs'):
463            list_configs(opts, _defaults)
464        else:
465            for config_file in opts.config_files():
466                b = build(config_file, _defaults = _defaults, opts = opts)
467                b.make()
468                del b
469    except error.general, gerr:
470        print gerr
471        sys.exit(1)
472    except error.internal, ierr:
473        print ierr
474        sys.exit(1)
475    except error.exit, eerr:
476        pass
477    except KeyboardInterrupt:
478        _notice(opts, 'user terminated')
479        sys.exit(1)
480    sys.exit(0)
481
482if __name__ == "__main__":
483    run(sys.argv)
Note: See TracBrowser for help on using the repository browser.