source: rtems-source-builder/source-builder/sb/build.py @ cafbcc6

4.104.114.95
Last change on this file since cafbcc6 was cafbcc6, checked in by Chris Johns <chrisj@…>, on 02/21/13 at 08:03:09

Support directly installing.

By default the Source Builder now directly installs in the prefix and
does not create tar files. You need to supply options to create build
set level tar files and/or package level tar files.
  • Property mode set to 100644
File size: 19.2 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2012 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package given a config file. It only builds to be
22# installed not to be package unless you run a packager around this.
23#
24
25import getopt
26import glob
27import os
28import shutil
29import stat
30import sys
31import urllib2
32import urlparse
33
34import check
35import config
36import defaults
37import error
38import execute
39import log
40import path
41
42#
43# Version of Sourcer Builder Build.
44#
45version = '0.1'
46
47def removeall(path):
48
49    def _onerror(function, path, excinfo):
50        print 'removeall error: (%r) %s' % (function, path)
51
52    shutil.rmtree(path, onerror = _onerror)
53    return
54
55def _notice(opts, text):
56    if not opts.quiet() and not log.default.has_stdout():
57        print text
58    log.output(text)
59    log.flush()
60
61class script:
62    """Create and manage a shell script."""
63
64    def __init__(self, quiet = True, trace = False):
65        self.quiet = quiet
66        self.trace = trace
67        self.reset()
68
69    def reset(self):
70        self.body = []
71        self.lc = 0
72
73    def append(self, text):
74        if type(text) is str:
75            text = text.splitlines()
76        if not self.quiet:
77            i = 0
78            for l in text:
79                i += 1
80                log.output('script:%3d: %s' % (self.lc + i, l))
81                if self.trace:
82                    print '%3d: S %s' % (self.lc + i,  l)
83        self.lc += len(text)
84        self.body.extend(text)
85
86    def write(self, name, check_for_errors = False):
87        s = None
88        try:
89            s = open(path.host(name), 'w')
90            s.write('\n'.join(self.body))
91            s.close()
92            os.chmod(path.host(name), stat.S_IRWXU | \
93                         stat.S_IRGRP | stat.S_IXGRP | \
94                         stat.S_IROTH | stat.S_IXOTH)
95        except IOError, err:
96            raise error.general('creating script: ' + name)
97        except:
98            if s is not None:
99                s.close()
100            raise
101        if s is not None:
102            s.close()
103
104class build:
105    """Build a package given a config file."""
106
107    def __init__(self, name, create_tar_files, _defaults, opts):
108        self.opts = opts
109        self.create_tar_files = create_tar_files
110        _notice(opts, 'building: ' + name)
111        self.config = config.file(name, _defaults = _defaults, opts = opts)
112        self.script = script(quiet = opts.quiet(), trace = opts.trace())
113
114    def _output(self, text):
115        if not self.opts.quiet():
116            log.output(text)
117
118    def rmdir(self, rmpath):
119        self._output('removing: %s' % (path.host(rmpath)))
120        if not self.opts.dry_run():
121            if path.exists(rmpath):
122                removeall(rmpath)
123
124    def mkdir(self, mkpath):
125        self._output('making dir: %s' % (path.host(mkpath)))
126        if not self.opts.dry_run():
127            if os.name == 'nt':
128                try:
129                    os.makedirs(path.host(mkpath))
130                except IOError, err:
131                    _notice(self.opts, 'warning: cannot make directory: %s' % (mkpath))
132                except OSError, err:
133                    _notice(self.opts, 'warning: cannot make directory: %s' % (mkpath))
134                except WindowsError, err:
135                    _notice(self.opts, 'warning: cannot make directory: %s' % (mkpath))
136            else:
137                try:
138                    os.makedirs(path.host(mkpath))
139                except IOError, err:
140                    _notice(self.opts, 'warning: cannot make directory: %s' % (mkpath))
141                except OSError, err:
142                    _notice(self.opts, 'warning: cannot make directory: %s' % (mkpath))
143
144    def get_file(self, url, local):
145        if local is None:
146            raise error.general('source/patch path invalid')
147        if not path.isdir(path.dirname(local)):
148            _notice(self.opts,
149                    'Creating source directory: %s' % (os.path.relpath(path.host(path.dirname(local)))))
150            self.mkdir(path.host(path.dirname(local)))
151        if not path.exists(local):
152            #
153            # Not localy found so we need to download it. Check if a URL has
154            # been provided on the command line.
155            #
156            url_bases = self.opts.urls()
157            urls = []
158            if url_bases is not None:
159                for base in url_bases:
160                    if base[-1:] != '/':
161                        base += '/'
162                    url_path = urlparse.urlsplit(url)[2]
163                    slash = url_path.rfind('/')
164                    if slash < 0:
165                        url_file = url_path
166                    else:
167                        url_file = url_path[slash + 1:]
168                    urls.append(urlparse.urljoin(base, url_file))
169            urls.append(url)
170            if self.opts.trace():
171                print '_url:', ','.join(urls), '->', local
172            for url in urls:
173                #
174                # Hack for GitHub.
175                #
176                if url.startswith('https://api.github.com'):
177                    url = urlparse.urljoin(url, self.config.expand('tarball/%{version}'))
178                _notice(self.opts, 'download: %s -> %s' % (url, os.path.relpath(path.host(local))))
179                if not self.opts.dry_run():
180                    failed = False
181                    _in = None
182                    _out = None
183                    try:
184                        _in = urllib2.urlopen(url)
185                        _out = open(path.host(local), 'wb')
186                        _out.write(_in.read())
187                    except IOError, err:
188                        msg = 'download: %s: error: %s' % (url, str(err))
189                        _notice(self.opts, msg)
190                        if path.exists(local):
191                            os.remove(path.host(local))
192                        failed = True
193                    except ValueError, err:
194                        msg = 'download: %s: error: %s' % (url, str(err))
195                        _notice(self.opts, msg)
196                        if path.exists(local):
197                            os.remove(path.host(local))
198                        failed = True
199                    except:
200                        msg = 'download: %s: error' % (url)
201                        print >> sys.stderr, msg
202                        if _out is not None:
203                            _out.close()
204                        raise
205                    if _out is not None:
206                        _out.close()
207                    if _in is not None:
208                        del _in
209                    if not failed:
210                        if not path.isfile(local):
211                            raise error.general('source is not a file: %s' % (path.host(local)))
212                        return
213            if not self.opts.dry_run():
214                raise error.general('downloading %s: all paths have failed, giving up' % (url))
215
216    def parse_url(self, url, pathkey):
217        #
218        # Split the source up into the parts we need.
219        #
220        source = {}
221        source['url'] = url
222        source['path'] = path.dirname(url)
223        source['file'] = path.basename(url)
224        source['name'], source['ext'] = path.splitext(source['file'])
225        #
226        # Get the file. Checks the local source directory first.
227        #
228        source['local'] = None
229        for p in self.config.define(pathkey).split(':'):
230            local = path.join(path.abspath(p), source['file'])
231            if source['local'] is None:
232                source['local'] = local
233            if path.exists(local):
234                source['local'] = local
235                break
236        #
237        # Is the file compressed ?
238        #
239        esl = source['ext'].split('.')
240        if esl[-1:][0] == 'gz':
241            source['compressed'] = '%{__gzip} -dc'
242        elif esl[-1:][0] == 'bz2':
243            source['compressed'] = '%{__bzip2} -dc'
244        elif esl[-1:][0] == 'bz2':
245            source['compressed'] = '%{__zip} -u'
246        elif esl[-1:][0] == 'xz':
247            source['compressed'] = '%{__xz} -dc'
248        source['script'] = ''
249        return source
250
251    def source(self, package, source_tag):
252        #
253        # Scan the sources found in the config file for the one we are
254        # after. Infos or tags are lists.
255        #
256        sources = package.sources()
257        url = None
258        for s in sources:
259            tag = s[len('source'):]
260            if tag.isdigit():
261                if int(tag) == source_tag:
262                    url = sources[s][0]
263                    break
264        if url is None:
265            raise error.general('source tag not found: source%d' % (source_tag))
266        source = self.parse_url(url, '_sourcedir')
267        self.get_file(source['url'], source['local'])
268        if 'compressed' in source:
269            source['script'] = source['compressed'] + ' ' + \
270                source['local'] + ' | %{__tar_extract} -'
271        else:
272            source['script'] = '%{__tar_extract} ' + source['local']
273        return source
274
275    def patch(self, package, args):
276        #
277        # Scan the patches found in the config file for the one we are
278        # after. Infos or tags are lists.
279        #
280        patches = package.patches()
281        url = None
282        for p in patches:
283            if args[0][1:].lower() == p:
284                url = patches[p][0]
285                break
286        if url is None:
287            raise error.general('patch tag not found: %s' % (args[0]))
288        #
289        # Parse the URL first in the source builder's patch directory.
290        #
291        patch = self.parse_url(url, '_patchdir')
292        #
293        # If not in the source builder package check the source directory.
294        #
295        if not path.exists(patch['local']):
296            patch = self.parse_url(url, '_sourcedir')
297        self.get_file(patch['url'], patch['local'])
298        if 'compressed' in patch:
299            patch['script'] = patch['compressed'] + ' ' +  patch['local']
300        else:
301            patch['script'] = '%{__cat} ' + patch['local']
302        patch['script'] += ' | %{__patch} ' + ' '.join(args[1:])
303        self.script.append(self.config.expand(patch['script']))
304
305    def setup(self, package, args):
306        self._output('prep: %s: %s' % (package.name(), ' '.join(args)))
307        opts, args = getopt.getopt(args[1:], 'qDcTn:b:a:')
308        source_tag = 0
309        quiet = False
310        unpack_default_source = True
311        delete_before_unpack = True
312        create_dir = False
313        name = None
314        unpack_before_chdir = True
315        for o in opts:
316            if o[0] == '-q':
317                quiet = True
318            elif o[0] == '-D':
319                delete_before_unpack = False
320            elif o[0] == '-c':
321                create_dir = True
322            elif o[0] == '-T':
323                unpack_default_source = False
324            elif o[0] == '-n':
325                name = o[1]
326            elif o[0] == '-b':
327                unpack_before_chdir = True
328                if not o[1].isdigit():
329                    raise error.general('setup source tag no a number: %s' % (o[1]))
330                source_tag = int(o[1])
331            elif o[0] == '-a':
332                unpack_before_chdir = False
333                source_tag = int(o[1])
334        source0 = None
335        source = self.source(package, source_tag)
336        if name is None:
337            if source:
338                name = source['name']
339            else:
340                name = source0['name']
341        self.script.append(self.config.expand('cd %{_builddir}'))
342        if delete_before_unpack:
343            self.script.append(self.config.expand('%{__rm} -rf ' + name))
344        if create_dir:
345            self.script.append(self.config.expand('%{__mkdir_p} ' + name))
346        #
347        # If -a? then change directory before unpacking.
348        #
349        if not unpack_before_chdir:
350            self.script.append(self.config.expand('cd ' + name))
351        #
352        # Unpacking the source. Note, treated the same as -a0.
353        #
354        if unpack_default_source and source_tag != 0:
355            source0 = self.source(package, 0)
356            if source0 is None:
357                raise error.general('no setup source0 tag found')
358            self.script.append(self.config.expand(source0['script']))
359        self.script.append(self.config.expand(source['script']))
360        if unpack_before_chdir:
361            self.script.append(self.config.expand('cd ' + name))
362        self.script.append(self.config.expand('%{__setup_post}'))
363        if create_dir:
364            self.script.append(self.config.expand('cd ..'))
365
366    def run(self, command, shell_opts = '', cwd = None):
367        e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
368        cmd = self.config.expand('%{___build_shell} -ex ' + shell_opts + ' ' + command)
369        self._output('run: ' + cmd)
370        exit_code, proc, output = e.shell(cmd, cwd = path.host(cwd))
371        if exit_code != 0:
372            raise error.general('shell cmd failed: %s' % (os.path.relpath(cmd)))
373
374    def builddir(self):
375        builddir = self.config.abspath('_builddir')
376        self.rmdir(builddir)
377        if not self.opts.dry_run():
378            self.mkdir(builddir)
379
380    def prep(self, package):
381        self.script.append('echo "==> %prep:"')
382        _prep = package.prep()
383        for l in _prep:
384            args = l.split()
385            if args[0] == '%setup':
386                self.setup(package, args)
387            elif args[0].startswith('%patch'):
388                self.patch(package, args)
389            else:
390                self.script.append(' '.join(args))
391
392    def build(self, package):
393        self.script.append('echo "==> %build:"')
394        _build = package.build()
395        for l in _build:
396            args = l.split()
397            self.script.append(' '.join(args))
398
399    def install(self, package):
400        self.script.append('echo "==> %install:"')
401        _install = package.install()
402        for l in _install:
403            args = l.split()
404            self.script.append(' '.join(args))
405
406    def files(self, package):
407        if self.create_tar_files:
408            self.script.append('echo "==> %files:"')
409            prefixbase = self.opts.prefixbase()
410            if prefixbase is None:
411                prefixbase = ''
412            inpath = path.join('%{buildroot}', prefixbase)
413            tardir = path.abspath(self.config.expand('%{_tardir}'))
414            self.script.append(self.config.expand('if test -d %s; then' % (inpath)))
415            self.script.append('  mkdir -p %s' % tardir)
416            self.script.append(self.config.expand('  cd ' + inpath))
417            tar = path.join(tardir, package.long_name() + '.tar.bz2')
418            cmd = self.config.expand('  %{__tar} -cf - . ' + '| %{__bzip2} > ' + tar)
419            self.script.append(cmd)
420            self.script.append(self.config.expand('  cd %{_builddir}'))
421            self.script.append('fi')
422
423    def clean(self, package):
424        self.script.append('echo "==> %clean:"')
425        _clean = package.clean()
426        if _clean is not None:
427            for l in _clean:
428                args = l.split()
429                self.script.append(' '.join(args))
430
431    def cleanup(self):
432        if not self.opts.no_clean():
433            buildroot = self.config.abspath('buildroot')
434            builddir = self.config.abspath('_builddir')
435            tmproot = self.config.abspath('_tmproot')
436            if self.opts.trace():
437                _notice(self.opts, 'cleanup: %s' % (buildroot))
438            self.rmdir(buildroot)
439            if self.opts.trace():
440                _notice(self.opts, 'cleanup: %s' % (builddir))
441            self.rmdir(builddir)
442            if self.opts.trace():
443                _notice(self.opts, 'cleanup: %s' % (tmproot))
444            self.rmdir(tmproot)
445
446    def make(self):
447        packages = self.config.packages()
448        package = packages['main']
449        name = package.name()
450        _notice(self.opts, 'package: %s' % (name))
451        self.script.reset()
452        self.script.append(self.config.expand('%{___build_template}'))
453        self.script.append('echo "=> ' + name + ':"')
454        self.prep(package)
455        self.build(package)
456        self.install(package)
457        self.files(package)
458        if not self.opts.no_clean():
459            self.clean(package)
460        if not self.opts.dry_run():
461            self.builddir()
462            sn = path.join(self.config.expand('%{_builddir}'), 'doit')
463            self._output('write script: ' + sn)
464            self.script.write(sn)
465            _notice(self.opts, 'building: ' + name)
466            self.run(sn)
467
468    def name(self):
469        packages = self.config.packages()
470        package = packages['main']
471        return package.name()
472
473def get_configs(opts, _defaults):
474
475    def _scan(_path, ext):
476        configs = []
477        for root, dirs, files in os.walk(_path):
478            prefix = root[len(_path) + 1:]
479            for file in files:
480                for e in ext:
481                    if file.endswith(e):
482                        configs += [path.join(prefix, file)]
483        return configs
484
485    configs = { 'paths': [], 'files': [] }
486    for cp in opts.expand('%{_configdir}', _defaults).split(':'):
487        configs['paths'] += [path.host(path.abspath(cp))]
488        configs['files'] += _scan(cp, ['.cfg', '.bset'])
489    configs['files'] = sorted(configs['files'])
490    return configs
491
492def run(args):
493    try:
494        optargs = { '--list-configs': 'List available configurations' }
495        opts, _defaults = defaults.load(args, optargs)
496        log.default = log.log(opts.logfiles())
497        _notice(opts, 'Source Builder, Package Builder v%s' % (version))
498        if not check.host_setup(opts, _defaults):
499            if not opts.force():
500                raise error.general('host build environment is not set up correctly (use --force to proceed)')
501            _notice(opts, 'warning: forcing build with known host setup problems')
502        if opts.get_arg('--list-configs'):
503            configs = get_configs(opts, _defaults)
504            for p in configs['paths']:
505                print 'Examining: %s' % (os.path.relpath(p))
506                for c in configs['files']:
507                    if c.endswith('.cfg'):
508                        print '    %s' % (c)
509        else:
510            for config_file in opts.config_files():
511                b = build(config_file, True, _defaults = _defaults, opts = opts)
512                b.make()
513                del b
514    except error.general, gerr:
515        print gerr
516        sys.exit(1)
517    except error.internal, ierr:
518        print ierr
519        sys.exit(1)
520    except error.exit, eerr:
521        pass
522    except KeyboardInterrupt:
523        _notice(opts, 'user terminated')
524        sys.exit(1)
525    sys.exit(0)
526
527if __name__ == "__main__":
528    run(sys.argv)
Note: See TracBrowser for help on using the repository browser.