source: rtems-source-builder/source-builder/sb/build.py @ 1b7e392

4.104.114.95
Last change on this file since 1b7e392 was 1b7e392, checked in by Chris Johns <chrisj@…>, on 02/20/13 at 23:16:52

Make the shell cmd path relative on error.
  • Property mode set to 100644
File size: 18.6 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2012 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package given a config file. It only builds to be
22# installed not to be package unless you run a packager around this.
23#
24
25import getopt
26import glob
27import os
28import shutil
29import stat
30import sys
31import urllib2
32import urlparse
33
34import check
35import config
36import defaults
37import error
38import execute
39import log
40import path
41
42#
43# Version of Sourcer Builder Build.
44#
45version = '0.1'
46
47def removeall(path):
48
49    def _onerror(function, path, excinfo):
50        print 'removeall error: (%r) %s' % (function, path)
51
52    shutil.rmtree(path, onerror = _onerror)
53    return
54
55def _notice(opts, text):
56    if not opts.quiet() and not log.default.has_stdout():
57        print text
58    log.output(text)
59    log.flush()
60
61class script:
62    """Create and manage a shell script."""
63
64    def __init__(self, quiet = True):
65        self.quiet = quiet
66        self.reset()
67
68    def reset(self):
69        self.body = []
70        self.lc = 0
71
72    def append(self, text):
73        if type(text) is str:
74            text = text.splitlines()
75        if not self.quiet:
76            i = 0
77            for l in text:
78                i += 1
79                log.output('script:%3d: ' % (self.lc + i) + l)
80        self.lc += len(text)
81        self.body.extend(text)
82
83    def write(self, name, check_for_errors = False):
84        s = None
85        try:
86            s = open(path.host(name), 'w')
87            s.write('\n'.join(self.body))
88            s.close()
89            os.chmod(path.host(name), stat.S_IRWXU | \
90                         stat.S_IRGRP | stat.S_IXGRP | \
91                         stat.S_IROTH | stat.S_IXOTH)
92        except IOError, err:
93            raise error.general('creating script: ' + name)
94        except:
95            if s is not None:
96                s.close()
97            raise
98        if s is not None:
99            s.close()
100
101class build:
102    """Build a package given a config file."""
103
104    def __init__(self, name, _defaults, opts):
105        self.opts = opts
106        _notice(opts, 'building: ' + name)
107        self.config = config.file(name, _defaults = _defaults, opts = opts)
108        self.script = script(quiet = opts.quiet())
109
110    def _output(self, text):
111        if not self.opts.quiet():
112            log.output(text)
113
114    def rmdir(self, rmpath):
115        self._output('removing: %s' % (path.host(rmpath)))
116        if not self.opts.dry_run():
117            if path.exists(rmpath):
118                removeall(rmpath)
119
120    def mkdir(self, mkpath):
121        self._output('making dir: %s' % (path.host(mkpath)))
122        if not self.opts.dry_run():
123            if os.name == 'nt':
124                try:
125                    os.makedirs(path.host(mkpath))
126                except IOError, err:
127                    _notice(self.opts, 'warning: cannot make directory: %s' % (mkpath))
128                except OSError, err:
129                    _notice(self.opts, 'warning: cannot make directory: %s' % (mkpath))
130                except WindowsError, err:
131                    _notice(self.opts, 'warning: cannot make directory: %s' % (mkpath))
132            else:
133                try:
134                    os.makedirs(path.host(mkpath))
135                except IOError, err:
136                    _notice(self.opts, 'warning: cannot make directory: %s' % (mkpath))
137                except OSError, err:
138                    _notice(self.opts, 'warning: cannot make directory: %s' % (mkpath))
139
140    def get_file(self, url, local):
141        if local is None:
142            raise error.general('source/patch path invalid')
143        if not path.isdir(path.dirname(local)):
144            _notice(self.opts,
145                    'Creating source directory: %s' % (os.path.relpath(path.host(path.dirname(local)))))
146            self.mkdir(path.host(path.dirname(local)))
147        if not path.exists(local):
148            #
149            # Not localy found so we need to download it. Check if a URL has
150            # been provided on the command line.
151            #
152            url_bases = self.opts.urls()
153            urls = []
154            if url_bases is not None:
155                for base in url_bases:
156                    if base[-1:] != '/':
157                        base += '/'
158                    url_path = urlparse.urlsplit(url)[2]
159                    slash = url_path.rfind('/')
160                    if slash < 0:
161                        url_file = url_path
162                    else:
163                        url_file = url_path[slash + 1:]
164                    urls.append(urlparse.urljoin(base, url_file))
165            urls.append(url)
166            if self.opts.trace():
167                print '_url:', ','.join(urls), '->', local
168            for url in urls:
169                #
170                # Hack for GitHub.
171                #
172                if url.startswith('https://api.github.com'):
173                    url = urlparse.urljoin(url, self.config.expand('tarball/%{version}'))
174                _notice(self.opts, 'download: %s -> %s' % (url, os.path.relpath(path.host(local))))
175                if not self.opts.dry_run():
176                    failed = False
177                    _in = None
178                    _out = None
179                    try:
180                        _in = urllib2.urlopen(url)
181                        _out = open(path.host(local), 'wb')
182                        _out.write(_in.read())
183                    except IOError, err:
184                        msg = 'download: %s: error: %s' % (url, str(err))
185                        _notice(self.opts, msg)
186                        if path.exists(local):
187                            os.remove(path.host(local))
188                        failed = True
189                    except ValueError, err:
190                        msg = 'download: %s: error: %s' % (url, str(err))
191                        _notice(self.opts, msg)
192                        if path.exists(local):
193                            os.remove(path.host(local))
194                        failed = True
195                    except:
196                        msg = 'download: %s: error' % (url)
197                        print >> sys.stderr, msg
198                        if _out is not None:
199                            _out.close()
200                        raise
201                    if _out is not None:
202                        _out.close()
203                    if _in is not None:
204                        del _in
205                    if not failed:
206                        if not path.isfile(local):
207                            raise error.general('source is not a file: %s' % (path.host(local)))
208                        return
209            if not self.opts.dry_run():
210                raise error.general('downloading %s: all paths have failed, giving up' % (url))
211
212    def parse_url(self, url, pathkey):
213        #
214        # Split the source up into the parts we need.
215        #
216        source = {}
217        source['url'] = url
218        source['path'] = path.dirname(url)
219        source['file'] = path.basename(url)
220        source['name'], source['ext'] = path.splitext(source['file'])
221        #
222        # Get the file. Checks the local source directory first.
223        #
224        source['local'] = None
225        for p in self.config.define(pathkey).split(':'):
226            local = path.join(path.abspath(p), source['file'])
227            if source['local'] is None:
228                source['local'] = local
229            if path.exists(local):
230                source['local'] = local
231                break
232        #
233        # Is the file compressed ?
234        #
235        esl = source['ext'].split('.')
236        if esl[-1:][0] == 'gz':
237            source['compressed'] = '%{__gzip} -dc'
238        elif esl[-1:][0] == 'bz2':
239            source['compressed'] = '%{__bzip2} -dc'
240        elif esl[-1:][0] == 'bz2':
241            source['compressed'] = '%{__zip} -u'
242        elif esl[-1:][0] == 'xz':
243            source['compressed'] = '%{__xz} -dc'
244        source['script'] = ''
245        return source
246
247    def source(self, package, source_tag):
248        #
249        # Scan the sources found in the config file for the one we are
250        # after. Infos or tags are lists.
251        #
252        sources = package.sources()
253        url = None
254        for s in sources:
255            tag = s[len('source'):]
256            if tag.isdigit():
257                if int(tag) == source_tag:
258                    url = sources[s][0]
259                    break
260        if url is None:
261            raise error.general('source tag not found: source%d' % (source_tag))
262        source = self.parse_url(url, '_sourcedir')
263        self.get_file(source['url'], source['local'])
264        if 'compressed' in source:
265            source['script'] = source['compressed'] + ' ' + \
266                source['local'] + ' | %{__tar_extract} -'
267        else:
268            source['script'] = '%{__tar_extract} ' + source['local']
269        return source
270
271    def patch(self, package, args):
272        #
273        # Scan the patches found in the config file for the one we are
274        # after. Infos or tags are lists.
275        #
276        patches = package.patches()
277        url = None
278        for p in patches:
279            if args[0][1:].lower() == p:
280                url = patches[p][0]
281                break
282        if url is None:
283            raise error.general('patch tag not found: %s' % (args[0]))
284        #
285        # Parse the URL first in the source builder's patch directory.
286        #
287        patch = self.parse_url(url, '_patchdir')
288        #
289        # If not in the source builder package check the source directory.
290        #
291        if not path.exists(patch['local']):
292            patch = self.parse_url(url, '_sourcedir')
293        self.get_file(patch['url'], patch['local'])
294        if 'compressed' in patch:
295            patch['script'] = patch['compressed'] + ' ' +  patch['local']
296        else:
297            patch['script'] = '%{__cat} ' + patch['local']
298        patch['script'] += ' | %{__patch} ' + ' '.join(args[1:])
299        self.script.append(self.config.expand(patch['script']))
300
301    def setup(self, package, args):
302        self._output('prep: %s: %s' % (package.name(), ' '.join(args)))
303        opts, args = getopt.getopt(args[1:], 'qDcTn:b:a:')
304        source_tag = 0
305        quiet = False
306        unpack_default_source = True
307        delete_before_unpack = True
308        create_dir = False
309        name = None
310        unpack_before_chdir = True
311        for o in opts:
312            if o[0] == '-q':
313                quiet = True
314            elif o[0] == '-D':
315                delete_before_unpack = False
316            elif o[0] == '-c':
317                create_dir = True
318            elif o[0] == '-T':
319                unpack_default_source = False
320            elif o[0] == '-n':
321                name = o[1]
322            elif o[0] == '-b':
323                unpack_before_chdir = True
324                if not o[1].isdigit():
325                    raise error.general('setup source tag no a number: %s' % (o[1]))
326                source_tag = int(o[1])
327            elif o[0] == '-a':
328                unpack_before_chdir = False
329                source_tag = int(o[1])
330        source0 = None
331        source = self.source(package, source_tag)
332        if name is None:
333            if source:
334                name = source['name']
335            else:
336                name = source0['name']
337        self.script.append(self.config.expand('cd %{_builddir}'))
338        if delete_before_unpack:
339            self.script.append(self.config.expand('%{__rm} -rf ' + name))
340        if create_dir:
341            self.script.append(self.config.expand('%{__mkdir_p} ' + name))
342        #
343        # If -a? then change directory before unpacking.
344        #
345        if not unpack_before_chdir:
346            self.script.append(self.config.expand('cd ' + name))
347        #
348        # Unpacking the source. Note, treated the same as -a0.
349        #
350        if unpack_default_source and source_tag != 0:
351            source0 = self.source(package, 0)
352            if source0 is None:
353                raise error.general('no setup source0 tag found')
354            self.script.append(self.config.expand(source0['script']))
355        self.script.append(self.config.expand(source['script']))
356        if unpack_before_chdir:
357            self.script.append(self.config.expand('cd ' + name))
358        self.script.append(self.config.expand('%{__setup_post}'))
359        if create_dir:
360            self.script.append(self.config.expand('cd ..'))
361
362    def run(self, command, shell_opts = '', cwd = None):
363        e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
364        cmd = self.config.expand('%{___build_shell} -ex ' + shell_opts + ' ' + command)
365        self._output('run: ' + cmd)
366        exit_code, proc, output = e.shell(cmd, cwd = path.host(cwd))
367        if exit_code != 0:
368            raise error.general('shell cmd failed: %s' % (os.path.relpath(cmd)))
369
370    def builddir(self):
371        builddir = self.config.abspath('_builddir')
372        self.rmdir(builddir)
373        if not self.opts.dry_run():
374            self.mkdir(builddir)
375
376    def prep(self, package):
377        self.script.append('echo "==> %prep:"')
378        _prep = package.prep()
379        for l in _prep:
380            args = l.split()
381            if args[0] == '%setup':
382                self.setup(package, args)
383            elif args[0].startswith('%patch'):
384                self.patch(package, args)
385            else:
386                self.script.append(' '.join(args))
387
388    def build(self, package):
389        self.script.append('echo "==> %build:"')
390        _build = package.build()
391        for l in _build:
392            args = l.split()
393            self.script.append(' '.join(args))
394
395    def install(self, package):
396        self.script.append('echo "==> %install:"')
397        _install = package.install()
398        for l in _install:
399            args = l.split()
400            self.script.append(' '.join(args))
401
402    def files(self, package):
403        self.script.append('echo "==> %files:"')
404        prefixbase = self.opts.prefixbase()
405        if prefixbase is None:
406            prefixbase = ''
407        inpath = path.join('%{buildroot}', prefixbase)
408        tardir = path.abspath(self.config.expand('%{_tardir}'))
409        self.script.append(self.config.expand('if test -d %s; then' % (inpath)))
410        self.script.append('  mkdir -p %s' % tardir)
411        self.script.append(self.config.expand('  cd ' + inpath))
412        tar = path.join(tardir, package.long_name() + '.tar.bz2')
413        cmd = self.config.expand('  %{__tar} -cf - . ' + '| %{__bzip2} > ' + tar)
414        self.script.append(cmd)
415        self.script.append(self.config.expand('  cd %{_builddir}'))
416        self.script.append('fi')
417
418    def clean(self, package):
419        self.script.append('echo "==> %clean:"')
420        _clean = package.clean()
421        if _clean is not None:
422            for l in _clean:
423                args = l.split()
424                self.script.append(' '.join(args))
425
426    def cleanup(self):
427        if not self.opts.no_clean():
428            buildroot = self.config.abspath('buildroot')
429            builddir = self.config.abspath('_builddir')
430            tmproot = self.config.abspath('_tmproot')
431            if self.opts.trace():
432                _notice(self.opts, 'cleanup: %s' % (buildroot))
433            self.rmdir(buildroot)
434            if self.opts.trace():
435                _notice(self.opts, 'cleanup: %s' % (builddir))
436            self.rmdir(builddir)
437            if self.opts.trace():
438                _notice(self.opts, 'cleanup: %s' % (tmproot))
439            self.rmdir(tmproot)
440
441    def make(self):
442        packages = self.config.packages()
443        package = packages['main']
444        name = package.name()
445        _notice(self.opts, 'package: %s' % (name))
446        self.script.reset()
447        self.script.append(self.config.expand('%{___build_template}'))
448        self.script.append('echo "=> ' + name + ':"')
449        self.prep(package)
450        self.build(package)
451        self.install(package)
452        self.files(package)
453        if not self.opts.no_clean():
454            self.clean(package)
455        if not self.opts.dry_run():
456            self.builddir()
457            sn = path.join(self.config.expand('%{_builddir}'), 'doit')
458            self._output('write script: ' + sn)
459            self.script.write(sn)
460            _notice(self.opts, 'building: ' + name)
461            self.run(sn)
462
463    def name(self):
464        packages = self.config.packages()
465        package = packages['main']
466        return package.name()
467
468def get_configs(opts, _defaults):
469
470    def _scan(_path, ext):
471        configs = []
472        for root, dirs, files in os.walk(_path):
473            prefix = root[len(_path) + 1:]
474            for file in files:
475                for e in ext:
476                    if file.endswith(e):
477                        configs += [path.join(prefix, file)]
478        return configs
479
480    configs = { 'paths': [], 'files': [] }
481    for cp in opts.expand('%{_configdir}', _defaults).split(':'):
482        configs['paths'] += [path.host(path.abspath(cp))]
483        configs['files'] += _scan(cp, ['.cfg', '.bset'])
484    configs['files'] = sorted(configs['files'])
485    return configs
486
487def run(args):
488    try:
489        optargs = { '--list-configs': 'List available configurations' }
490        opts, _defaults = defaults.load(args, optargs)
491        log.default = log.log(opts.logfiles())
492        _notice(opts, 'Source Builder, Package Builder v%s' % (version))
493        if not check.host_setup(opts, _defaults):
494            if not opts.force():
495                raise error.general('host build environment is not set up correctly (use --force to proceed)')
496            _notice(opts, 'warning: forcing build with known host setup problems')
497        if opts.get_arg('--list-configs'):
498            list_configs(opts, _defaults)
499        else:
500            for config_file in opts.config_files():
501                b = build(config_file, _defaults = _defaults, opts = opts)
502                b.make()
503                del b
504    except error.general, gerr:
505        print gerr
506        sys.exit(1)
507    except error.internal, ierr:
508        print ierr
509        sys.exit(1)
510    except error.exit, eerr:
511        pass
512    except KeyboardInterrupt:
513        _notice(opts, 'user terminated')
514        sys.exit(1)
515    sys.exit(0)
516
517if __name__ == "__main__":
518    run(sys.argv)
Note: See TracBrowser for help on using the repository browser.