source: rtems-source-builder/source-builder/sb/build.py @ 38fd56c

5
Last change on this file since 38fd56c was 38fd56c, checked in by Chris Johns <chrisj@…>, on 09/27/18 at 21:27:57

sb: Monitor the build disk usage. Report the usage, total and various sizes

  • Track the size of a build of a package in a build set to determine the maximum amout of disk space used. This can be used as a guide to documenting how much space a user needs to set aside to build a specific set of tools.
  • The %clean stage of a build is now split into a separate script. I do not think this is an issue because I could not find any %clean sections in any build configs we have. In time support for the %clean section will be removed, the package builder cleans up.

Closes #3516
  • Property mode set to 100644
File size: 24.9 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2018 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# This code builds a package given a config file. It only builds to be
22# installed not to be package unless you run a packager around this.
23#
24
25from __future__ import print_function
26
27import copy
28import getopt
29import glob
30import os
31import shutil
32import stat
33import sys
34
35try:
36    import check
37    import config
38    import download
39    import error
40    import ereport
41    import execute
42    import log
43    import options
44    import path
45    import sources
46    import version
47except KeyboardInterrupt:
48    print('abort: user terminated')
49    sys.exit(1)
50except:
51    print('error: unknown application load error')
52    sys.exit(1)
53
54def humanize_number(num, suffix):
55    for unit in ['','K','M','G','T','P','E','Z']:
56        if abs(num) < 1024.0:
57            return "%5.3f%s%s" % (num, unit, suffix)
58        num /= 1024.0
59    return "%.3f%s%s" % (size, 'Y', suffix)
60
61class script:
62    """Create and manage a shell script."""
63
64    def __init__(self):
65        self.reset()
66
67    def reset(self):
68        self.body = []
69        self.lc = 0
70
71    def append(self, text):
72        if type(text) is str:
73            text = text.splitlines()
74        if not log.quiet:
75            i = 0
76            for l in text:
77                i += 1
78                log.output('script:%3d: %s' % (self.lc + i, l))
79        self.lc += len(text)
80        self.body.extend(text)
81
82    def write(self, name, check_for_errors = False):
83        s = None
84        try:
85            s = open(path.host(name), 'w')
86            s.write('\n'.join(self.body))
87            s.close()
88            os.chmod(path.host(name), stat.S_IRWXU | \
89                         stat.S_IRGRP | stat.S_IXGRP | \
90                         stat.S_IROTH | stat.S_IXOTH)
91        except IOError as err:
92            raise error.general('creating script: ' + name)
93        except:
94            if s is not None:
95                s.close()
96            raise
97        if s is not None:
98            s.close()
99
100class build:
101    """Build a package given a config file."""
102
103    def _name_(self, name):
104        #
105        # If on Windows use shorter names to keep the build paths.
106        #
107        if options.host_windows:
108            buildname = ''
109            add = True
110            for c in name:
111                if c == '-':
112                    add = True
113                elif add:
114                    buildname += c
115                    add = False
116            return buildname
117        else:
118            return name
119
120    def _generate_report_(self, header, footer = None):
121        ereport.generate('rsb-report-%s.txt' % self.macros['name'],
122                         self.opts, header, footer)
123
124    def __init__(self, name, create_tar_files, opts, macros = None):
125        try:
126            self.opts = opts
127            self.init_name = name
128            self.init_macros = macros
129            self.config = None
130            self.create_tar_files = create_tar_files
131            log.notice('config: ' + name)
132            self.set_macros(macros)
133            self.config = config.file(name, opts, self.macros)
134            self.script_build = script()
135            self.script_clean = script()
136            self.macros['buildname'] = self._name_(self.macros['name'])
137        except error.general as gerr:
138            log.notice(str(gerr))
139            log.stderr('Build FAILED')
140            raise
141        except error.internal as ierr:
142            log.notice(str(ierr))
143            log.stderr('Internal Build FAILED')
144            raise
145        except:
146            raise
147
148    def copy_init_macros(self):
149        return copy.copy(self.init_macros)
150
151    def copy_macros(self):
152        return copy.copy(self.macros)
153
154    def set_macros(self, macros):
155        if macros is None:
156            self.macros = copy.copy(self.opts.defaults)
157        else:
158            self.macros = copy.copy(macros)
159        if self.config:
160            self.config.set_macros(self.macros)
161
162    def rmdir(self, rmpath):
163        log.output('removing: %s' % (path.host(rmpath)))
164        if not self.opts.dry_run():
165            if path.exists(rmpath):
166                path.removeall(rmpath)
167
168    def mkdir(self, mkpath):
169        log.output('making dir: %s' % (path.host(mkpath)))
170        if not self.opts.dry_run():
171            path.mkdir(mkpath)
172
173    def canadian_cross(self):
174        _host = self.config.expand('%{_host}')
175        _build = self.config.expand('%{_build}')
176        _target = self.config.expand('%{_target}')
177        _allowed = self.config.defined('%{allow_cxc}')
178        if len(_host) and len(_build) and (_target) and \
179           _allowed and _host != _build and _host != _target:
180            return True
181        return False
182
183    def installable(self):
184        _host = self.config.expand('%{_host}')
185        _build = self.config.expand('%{_build}')
186        _canadian_cross = self.canadian_cross()
187        if self.macros.get('_disable_installing') and \
188           self.config.expand('%{_disable_installing}') == 'yes':
189            _disable_installing = True
190        else:
191            _disable_installing = False
192        _no_install = self.opts.no_install()
193        log.trace('_build: installable: host=%s build=%s ' \
194                  'no-install=%r Cxc=%r disable_installing=%r disabled=%r' % \
195                  (_host, _build, _no_install, _canadian_cross, _disable_installing, \
196                   self.disabled()))
197        return len(_host) and len(_build) and \
198            not self.disabled() and \
199            not _disable_installing and \
200            not _canadian_cross
201
202    def source(self, name):
203        #
204        # Return the list of sources. Merge in any macro defined sources as
205        # these may be overridden by user loaded macros.
206        #
207        _map = 'source-%s' % (name)
208        src_keys = [s for s in self.macros.map_keys(_map) if s != 'setup']
209        if len(src_keys) == 0:
210            raise error.general('no source set: %s (%s)' % (name, _map))
211        srcs = []
212        for s in src_keys:
213            sm = self.macros.get(s, globals = False, maps = _map)
214            if sm is None:
215                raise error.internal('source macro not found: %s in %s (%s)' % \
216                                         (s, name, _map))
217            opts = []
218            url = []
219            for sp in sm[2].split():
220                if len(url) == 0 and sp[0] == '-':
221                    opts += [sp]
222                else:
223                    url += [sp]
224            if len(url) == 0:
225                raise error.general('source URL not found: %s' % (' '.join(args)))
226            #
227            # Look for --rsb-file as an option we use as a local file name.
228            # This can be used if a URL has no reasonable file name the
229            # download URL parser can figure out.
230            #
231            file_override = None
232            if len(opts) > 0:
233                for o in opts:
234                    if o.startswith('--rsb-file'):
235                       os_ = o.split('=')
236                       if len(os_) != 2:
237                           raise error.general('invalid --rsb-file option: %s' % (' '.join(args)))
238                       if os_[0] != '--rsb-file':
239                           raise error.general('invalid --rsb-file option: %s' % (' '.join(args)))
240                       file_override = os_[1]
241                opts = [o for o in opts if not o.startswith('--rsb-')]
242            url = self.config.expand(' '.join(url))
243            src = download.parse_url(url, '_sourcedir', self.config, self.opts, file_override)
244            download.get_file(src['url'], src['local'], self.opts, self.config)
245            if 'symlink' in src:
246                sname = name.replace('-', '_')
247                src['script'] = '%%{__ln_s} %s ${source_dir_%s}' % (src['symlink'], sname)
248            elif 'compressed' in src:
249                #
250                # Zip files unpack as well so do not use tar.
251                #
252                src['script'] = '%s %s' % (src['compressed'], src['local'])
253                if src['compressed-type'] != 'zip':
254                    src['script'] += ' | %{__tar_extract} -'
255            else:
256                src['script'] = '%%{__tar_extract} %s' % (src['local'])
257            srcs += [src]
258        return srcs
259
260    def source_setup(self, package, args):
261        log.output('source setup: %s: %s' % (package.name(), ' '.join(args)))
262        setup_name = args[1]
263        args = args[1:]
264        try:
265            opts, args = getopt.getopt(args[1:], 'qDcn:ba')
266        except getopt.GetoptError as ge:
267            raise error.general('source setup error: %s' % str(ge))
268        quiet = False
269        unpack_before_chdir = True
270        delete_before_unpack = True
271        create_dir = False
272        deleted_dir = False
273        created_dir = False
274        changed_dir = False
275        opt_name = None
276        for o in opts:
277            if o[0] == '-q':
278                quiet = True
279            elif o[0] == '-D':
280                delete_before_unpack = False
281            elif o[0] == '-c':
282                create_dir = True
283            elif o[0] == '-n':
284                opt_name = o[1]
285            elif o[0] == '-b':
286                unpack_before_chdir = True
287            elif o[0] == '-a':
288                unpack_before_chdir = False
289        name = None
290        for source in self.source(setup_name):
291            if name is None:
292                if opt_name is None:
293                    if source:
294                        opt_name = source['name']
295                    else:
296                        raise error.general('setup source tag not found: %d' % (source_tag))
297                else:
298                    name = opt_name
299            self.script_build.append(self.config.expand('cd %{_builddir}'))
300            if not deleted_dir and  delete_before_unpack:
301                self.script_build.append(self.config.expand('%{__rm} -rf ' + name))
302                deleted_dir = True
303            if not created_dir and create_dir:
304                self.script_build.append(self.config.expand('%{__mkdir_p} ' + name))
305                created_dir = True
306            if not changed_dir and (not unpack_before_chdir or create_dir):
307                self.script_build.append(self.config.expand('cd ' + name))
308                changed_dir = True
309            self.script_build.append(self.config.expand(source['script']))
310        if not changed_dir and (unpack_before_chdir and not create_dir):
311            self.script_build.append(self.config.expand('cd ' + name))
312            changed_dir = True
313        self.script_build.append(self.config.expand('%{__setup_post}'))
314
315    def patch_setup(self, package, args):
316        name = args[1]
317        args = args[2:]
318        _map = 'patch-%s' % (name)
319        default_opts = ' '.join(args)
320        patch_keys = [p for p in self.macros.map_keys(_map) if p != 'setup']
321        patches = []
322        for p in patch_keys:
323            pm = self.macros.get(p, globals = False, maps = _map)
324            if pm is None:
325                raise error.internal('patch macro not found: %s in %s (%s)' % \
326                                         (p, name, _map))
327            opts = []
328            url = []
329            for pp in pm[2].split():
330                if len(url) == 0 and pp[0] == '-':
331                    opts += [pp]
332                else:
333                    url += [pp]
334            if len(url) == 0:
335                raise error.general('patch URL not found: %s' % (' '.join(args)))
336            #
337            # Look for --rsb-file as an option we use as a local file name.
338            # This can be used if a URL has no reasonable file name the
339            # download URL parser can figure out.
340            #
341            file_override = None
342            if len(opts) > 0:
343                for o in opts:
344                    if o.startswith('--rsb-file'):
345                       os_ = o.split('=')
346                       if len(os_) != 2:
347                           raise error.general('invalid --rsb-file option: %s' % (' '.join(args)))
348                       if os_[0] != '--rsb-file':
349                           raise error.general('invalid --rsb-file option: %s' % (' '.join(args)))
350                       file_override = os_[1]
351                opts = [o for o in opts if not o.startswith('--rsb-')]
352            if len(opts) == 0:
353                opts = default_opts
354            else:
355                opts = ' '.join(opts)
356            opts = self.config.expand(opts)
357            url = self.config.expand(' '.join(url))
358            #
359            # Parse the URL first in the source builder's patch directory.
360            #
361            patch = download.parse_url(url, '_patchdir', self.config, self.opts, file_override)
362            #
363            # Download the patch
364            #
365            download.get_file(patch['url'], patch['local'], self.opts, self.config)
366            if 'compressed' in patch:
367                patch['script'] = patch['compressed'] + ' ' +  patch['local']
368            else:
369                patch['script'] = '%{__cat} ' + patch['local']
370            patch['script'] += ' | %%{__patch} %s' % (opts)
371            self.script_build.append(self.config.expand(patch['script']))
372
373    def run(self, command, shell_opts = '', cwd = None):
374        e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
375        cmd = self.config.expand('%{___build_shell} -ex ' + shell_opts + ' ' + command)
376        log.output('run: ' + cmd)
377        exit_code, proc, output = e.shell(cmd, cwd = path.host(cwd))
378        if exit_code != 0:
379            log.output('shell cmd failed: %s' % (cmd))
380            raise error.general('building %s' % (self.macros['buildname']))
381
382    def builddir(self):
383        builddir = self.config.abspath('_builddir')
384        self.rmdir(builddir)
385        if not self.opts.dry_run():
386            self.mkdir(builddir)
387
388    def prep(self, package):
389        self.script_build.append('echo "==> %prep:"')
390        _prep = package.prep()
391        if _prep:
392            for l in _prep:
393                args = l.split()
394                if len(args):
395                    def err(msg):
396                        raise error.general('%s: %s' % (package, msg))
397                    if args[0] == '%setup':
398                        if len(args) == 1:
399                            raise error.general('invalid %%setup directive: %s' % \
400                                                (' '.join(args)))
401                        if args[1] == 'source':
402                            self.source_setup(package, args[1:])
403                        elif args[1] == 'patch':
404                            self.patch_setup(package, args[1:])
405                    elif args[0] in ['%patch', '%source']:
406                        sources.process(args[0][1:], args[1:], self.macros, err)
407                    elif args[0] == '%hash':
408                        sources.hash(args[1:], self.macros, err)
409                        self.hash(package, args)
410                    else:
411                        self.script_build.append(' '.join(args))
412
413    def build(self, package):
414        self.script_build.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"')
415        self.script_build.append('%s ${SB_BUILD_ROOT}' %
416                                 (self.config.expand('%{__rmdir}')))
417        self.script_build.append('%s ${SB_BUILD_ROOT}' %
418                                 (self.config.expand('%{__mkdir_p}')))
419        self.script_build.append('echo "==> %build:"')
420        _build = package.build()
421        if _build:
422            for l in _build:
423                self.script_build.append(l)
424
425    def install(self, package):
426        self.script_build.append('echo "==> %install:"')
427        _install = package.install()
428        if _install:
429            for l in _install:
430                args = l.split()
431                self.script_build.append(' '.join(args))
432
433    def files(self, package):
434        if self.create_tar_files \
435           and not self.macros.get('%{_disable_packaging'):
436            self.script_build.append('echo "==> %files:"')
437            inpath = path.abspath(self.config.expand('%{buildroot}'))
438            tardir = path.abspath(self.config.expand('%{_tardir}'))
439            self.script_build.append(self.config.expand('if test -d %s; then' % (inpath)))
440            self.script_build.append(self.config.expand('  %%{__mkdir_p} %s' % tardir))
441            self.script_build.append(self.config.expand('  cd ' + inpath))
442            tar = path.join(tardir, package.long_name() + '.tar.bz2')
443            cmd = self.config.expand('  %{__tar} -cf - . ' + '| %{__bzip2} > ' + tar)
444            self.script_build.append(cmd)
445            self.script_build.append(self.config.expand('  cd %{_builddir}'))
446            self.script_build.append('fi')
447
448    def clean(self, package):
449        self.script_clean.reset()
450        self.script_clean.append(self.config.expand('%{___build_template}'))
451        self.script_clean.append('echo "=> ' + package.name() + ': CLEAN"')
452        self.script_clean.append('echo "==> %clean:"')
453        _clean = package.clean()
454        if _clean is not None:
455            for l in _clean:
456                args = l.split()
457                self.script_clean.append(' '.join(args))
458
459    def sizes(self, package):
460        def _sizes(package, what, path):
461            package.set_size(what, path)
462            s = humanize_number(package.get_size(what), 'B')
463            log.trace('size: %s (%s): %s (%d)' % (what, path, s, package.get_size(what)))
464            return s
465        s = {}
466        for p in [('build', '%{_builddir}'),
467                  ('build', '%{buildroot}'),
468                  ('installed', '%{buildroot}')]:
469            hs = _sizes(package, p[0], self.config.expand(p[1]))
470            s[p[0]] = hs
471        log.notice('sizes: %s: %s (installed: %s)' % (package.name(),
472                                                      s['build'],
473                                                      s['installed']))
474
475    def build_package(self, package):
476        if self.canadian_cross():
477            if not self.config.defined('%{allow_cxc}'):
478                raise error.general('Canadian Cross is not allowed')
479            self.script_build.append('echo "==> Candian-cross build/target:"')
480            self.script_build.append('SB_CXC="yes"')
481        else:
482            self.script_build.append('SB_CXC="no"')
483        self.build(package)
484        self.install(package)
485        self.files(package)
486        if not self.opts.no_clean():
487            self.clean(package)
488
489    def cleanup(self):
490        package = self.main_package()
491        if not package.disabled() and not self.opts.no_clean():
492            buildroot = self.config.abspath('buildroot')
493            builddir = self.config.abspath('_builddir')
494            buildcxcdir = self.config.abspath('_buildcxcdir')
495            tmproot = self.config.abspath('_tmproot')
496            log.trace('cleanup: %s' % (buildroot))
497            self.rmdir(buildroot)
498            log.trace('cleanup: %s' % (builddir))
499            self.rmdir(builddir)
500            if self.canadian_cross():
501                log.trace('cleanup: %s' % (buildcxcdir))
502                self.rmdir(buildcxcdir)
503            log.trace('cleanup: %s' % (tmproot))
504            self.rmdir(tmproot)
505
506    def main_package(self):
507        packages = self.config.packages()
508        return packages['main']
509
510    def reload(self):
511        self.config.load(self.init_name)
512
513    def make(self):
514        package = self.main_package()
515        if package.disabled():
516            log.notice('package: nothing to build')
517        else:
518            try:
519                name = package.name()
520                if self.canadian_cross():
521                    cxc_label = '(Cxc) '
522                else:
523                    cxc_label = ''
524                log.notice('package: %s%s' % (cxc_label, name))
525                log.trace('---- macro maps %s' % ('-' * 55))
526                log.trace('%s' % (str(self.config.macros)))
527                log.trace('-' * 70)
528                self.script_build.reset()
529                self.script_build.append(self.config.expand('%{___build_template}'))
530                self.script_build.append('echo "=> ' + name + ': BUILD"')
531                self.prep(package)
532                self.build_package(package)
533                if not self.opts.dry_run():
534                    self.builddir()
535                    build_sn = path.join(self.config.expand('%{_builddir}'), 'do-build')
536                    log.output('write script: ' + build_sn)
537                    self.script_build.write(build_sn)
538                    clean_sn = path.join(self.config.expand('%{_builddir}'), 'do-clean')
539                    log.output('write script: ' + clean_sn)
540                    self.script_clean.write(clean_sn)
541                    log.notice('building: %s%s' % (cxc_label, name))
542                    self.run(build_sn)
543                    self.sizes(package)
544                    log.notice('cleaning: %s%s' % (cxc_label, name))
545                    self.run(clean_sn)
546            except error.general as gerr:
547                log.notice(str(gerr))
548                log.stderr('Build FAILED')
549                self._generate_report_('Build: %s' % (gerr))
550                raise
551            except error.internal as ierr:
552                log.notice(str(ierr))
553                log.stderr('Internal Build FAILED')
554                self._generate_report_('Build: %s' % (ierr))
555                raise
556            except:
557                raise
558            if self.opts.dry_run():
559                self._generate_report_('Build: dry run (no actual error)',
560                                       'Build: dry run (no actual error)')
561
562    def name(self):
563        packages = self.config.packages()
564        package = packages['main']
565        return package.name()
566
567    def disabled(self):
568        packages = self.config.packages()
569        package = packages['main']
570        return package.disabled()
571
572    def get_build_size(self):
573        package = self.main_package()
574        if package.disabled():
575            return 0
576        return package.get_size('build')
577
578    def get_installed_size(self):
579        package = self.main_package()
580        if package.disabled():
581            return 0
582        return package.get_size('installed')
583
584def get_configs(opts):
585
586    def _scan(_path, ext):
587        configs = []
588        for root, dirs, files in os.walk(_path):
589            prefix = root[len(_path) + 1:]
590            for file in files:
591                for e in ext:
592                    if file.endswith(e):
593                        configs += [path.join(prefix, file)]
594        return configs
595
596    configs = { 'paths': [], 'files': [] }
597    for cp in opts.defaults.expand('%{_configdir}').split(':'):
598        hcp = path.host(path.abspath(cp))
599        configs['paths'] += [hcp]
600        configs['files'] += _scan(hcp, ['.cfg', '.bset'])
601    configs['files'] = sorted(set(configs['files']))
602    return configs
603
604def find_config(config, configs):
605    config_root, config_ext = path.splitext(config)
606    if config_ext not in ['', '.bset', '.cfg']:
607        config_root = config
608        config_ext = ''
609    for c in configs['files']:
610        r, e = path.splitext(c)
611        if config_root == r:
612            if config_ext == '' or config_ext == e:
613                return c
614    return None
615
616def run(args):
617    ec = 0
618    try:
619        optargs = { '--list-configs': 'List available configurations' }
620        opts = options.load(args, optargs)
621        log.notice('RTEMS Source Builder, Package Builder, %s' % (version.str()))
622        opts.log_info()
623        if not check.host_setup(opts):
624            if not opts.force():
625                raise error.general('host build environment is not set up' +
626                                    ' correctly (use --force to proceed)')
627            log.notice('warning: forcing build with known host setup problems')
628        if opts.get_arg('--list-configs'):
629            configs = get_configs(opts)
630            for p in configs['paths']:
631                print('Examining: %s' % (os.path.relpath(p)))
632                for c in configs['files']:
633                    if c.endswith('.cfg'):
634                        print('    %s' % (c))
635        else:
636            for config_file in opts.config_files():
637                b = build(config_file, True, opts)
638                b.make()
639                b = None
640    except error.general as gerr:
641        log.stderr('Build FAILED')
642        ec = 1
643    except error.internal as ierr:
644        log.stderr('Internal Build FAILED')
645        ec = 1
646    except error.exit as eerr:
647        pass
648    except KeyboardInterrupt:
649        log.notice('abort: user terminated')
650        ec = 1
651    sys.exit(ec)
652
653if __name__ == "__main__":
654    run(sys.argv)
Note: See TracBrowser for help on using the repository browser.