source: rtems-tools/tester/rt/check.py @ 056bd4b

5
Last change on this file since 056bd4b was 056bd4b, checked in by Chris Johns <chrisj@…>, on 03/15/17 at 04:23:40

rtems-bsp-builder: Fix profile error message.
  • Property mode set to 100755
File size: 24.1 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2016 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# 1. Redistributions of source code must retain the above copyright notice,
12# this list of conditions and the following disclaimer.
13#
14# 2. Redistributions in binary form must reproduce the above copyright notice,
15# this list of conditions and the following disclaimer in the documentation
16# and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31from __future__ import print_function
32
33import argparse
34import datetime
35import operator
36import os
37import sys
38
39try:
40    import configparser
41except:
42    import ConfigParser as configparser
43
44from rtemstoolkit import execute
45from rtemstoolkit import error
46from rtemstoolkit import log
47from rtemstoolkit import path
48from rtemstoolkit import version
49
50def rtems_version():
51    return version.version()
52
53class warnings_counter:
54
55    def __init__(self, rtems):
56        self.rtems = path.host(rtems)
57        self.reset()
58
59    def report(self):
60        str = ''
61        sw = sorted(self.warnings.items(), key = operator.itemgetter(1), reverse = True)
62        for w in sw:
63            str += ' %5d %s%s' % (w[1], w[0], os.linesep)
64        return str
65
66    def accumulate(self, total):
67        for w in self.warnings:
68            if w not in total.warnings:
69                total.warnings[w] = self.warnings[w]
70            else:
71                total.warnings[w] += self.warnings[w]
72        total.count += self.count
73
74    def get(self):
75        return self.count
76
77    def reset(self):
78        self.warnings = { }
79        self.count = 0
80
81    def output(self, text):
82        for l in text.splitlines():
83            if ' warning:' in l:
84                self.count += 1
85                ws = l.split(' ')
86                if len(ws) > 0:
87                    ws = ws[0].split(':')
88                    w = path.abspath(ws[0])
89                    w = w.replace(self.rtems, '')
90                    if path.isabspath(w):
91                        w = w[1:]
92                    #
93                    # Ignore compiler option warnings.
94                    #
95                    if len(ws) >= 3:
96                        w = '%s:%s:%s' % (w, ws[1], ws[2])
97                        if w not in self.warnings:
98                            self.warnings[w] = 0
99                        self.warnings[w] += 1
100        log.output(text)
101
102class results:
103
104    def __init__(self):
105        self.passes = []
106        self.fails = []
107
108    def _arch_bsp(self, arch, bsp):
109        return '%s/%s' % (arch, bsp)
110
111    def add(self, good, arch, bsp, configure, warnings):
112        if good:
113            self.passes += [(arch, bsp, configure, warnings)]
114        else:
115            self.fails += [(arch, bsp, configure, 0)]
116
117    def report(self):
118        log.notice('* Passes: %d   Failures: %d' %
119                   (len(self.passes), len(self.fails)))
120        log.output()
121        log.output('Build Report')
122        log.output('   Passes: %d   Failures: %d' %
123                   (len(self.passes), len(self.fails)))
124        log.output(' Failures:')
125        if len(self.fails) == 0:
126            log.output('None')
127        else:
128            max_col = 0
129            for f in self.fails:
130                arch_bsp = self._arch_bsp(f[0], f[1])
131                if len(arch_bsp) > max_col:
132                    max_col = len(arch_bsp)
133            for f in self.fails:
134                config_cmd = f[2]
135                config_at = config_cmd.find('configure')
136                if config_at != -1:
137                    config_cmd = config_cmd[config_at:]
138                log.output(' %*s:  %s' % (max_col + 2,
139                                          self._arch_bsp(f[0], f[1]),
140                                          config_cmd))
141        log.output(' Passes:')
142        if len(self.passes) == 0:
143            log.output('None')
144        else:
145            max_col = 0
146            for f in self.passes:
147                arch_bsp = self._arch_bsp(f[0], f[1])
148                if len(arch_bsp) > max_col:
149                    max_col = len(arch_bsp)
150            for f in self.passes:
151                config_cmd = f[2]
152                config_at = config_cmd.find('configure')
153                if config_at != -1:
154                    config_cmd = config_cmd[config_at:]
155                log.output(' %*s:  %5d  %s' % (max_col + 2,
156                                               self._arch_bsp(f[0], f[1]),
157                                               f[3],
158                                               config_cmd))
159
160class configuration:
161
162    def __init__(self):
163        self.config = configparser.ConfigParser()
164        self.name = None
165        self.archs = { }
166        self.builds = { }
167        self.profiles = { }
168
169    def __str__(self):
170        import pprint
171        s = self.name + os.linesep
172        s += 'Archs:' + os.linesep + \
173             pprint.pformat(self.archs, indent = 1, width = 80) + os.linesep
174        s += 'Builds:' + os.linesep + \
175             pprint.pformat(self.builds, indent = 1, width = 80) + os.linesep
176        s += 'Profiles:' + os.linesep + \
177             pprint.pformat(self.profiles, indent = 1, width = 80) + os.linesep
178        return s
179
180    def _get_item(self, section, label, err = True):
181        try:
182            rec = self.config.get(section, label).replace(os.linesep, ' ')
183            return rec
184        except:
185            if err:
186                raise error.general('config: no %s found in %s' % (label, section))
187        return None
188
189    def _get_items(self, section, err = True):
190        try:
191            items = self.config.items(section)
192            return items
193        except:
194            if err:
195                raise error.general('config: section %s not found' % (section))
196        return []
197
198    def _comma_list(self, section, label, error = True):
199        items = self._get_item(section, label, error)
200        if items is None:
201            return []
202        return sorted(set([a.strip() for a in items.split(',')]))
203
204    def load(self, name, variation):
205        if not path.exists(name):
206            raise error.general('config: cannot read configuration: %s' % (name))
207        self.name = name
208        try:
209            self.config.read(name)
210        except configparser.ParsingError as ce:
211            raise error.general('config: %s' % (ce))
212        archs = []
213        self.profiles['profiles'] = self._comma_list('profiles', 'profiles', error = False)
214        if len(self.profiles['profiles']) == 0:
215            self.profiles['profiles'] = ['tier-%d' % (t) for t in range(1,4)]
216        for p in self.profiles['profiles']:
217            profile = {}
218            profile['name'] = p
219            profile['archs'] = self._comma_list(profile['name'], 'archs')
220            archs += profile['archs']
221            for arch in profile['archs']:
222                bsps = 'bsps_%s' % (arch)
223                profile[bsps] = self._comma_list(profile['name'], bsps)
224            self.profiles[profile['name']] = profile
225        for a in set(archs):
226            arch = {}
227            arch['excludes'] = {}
228            for exclude in self._comma_list(a, 'exclude', error = False):
229                arch['excludes'][exclude] = ['all']
230            for i in self._get_items(a, False):
231                if i[0].startswith('exclude_'):
232                    exclude = i[0][len('exclude_'):]
233                    if exclude not in arch['excludes']:
234                        arch['excludes'][exclude] = []
235                    arch['excludes'][exclude] += sorted(set([b.strip() for b in i[1].split(',')]))
236            arch['bsps'] = self._comma_list(a, 'bsps', error = False)
237            for b in arch['bsps']:
238                arch[b] = {}
239                arch[b]['bspopts'] = self._comma_list(a, 'bspopts_%s' % (b), error = False)
240            self.archs[a] = arch
241        builds = {}
242        builds['default'] = self._get_item('builds', 'default').split()
243        builds['variations'] = self._comma_list('builds', 'variations')
244        if variation is None:
245            variation = builds['default']
246        builds['variation'] = variation
247        builds['base'] = self._get_item('builds', 'standard').split()
248        builds['variations'] = self._comma_list('builds', variation)
249        builds['var_options'] = {}
250        for v in builds['variations']:
251            if v == 'base':
252                builds['var_options'][v] = self._get_item('builds', v).split()
253            else:
254                builds['var_options'][v] = []
255        self.builds = builds
256
257    def variations(self):
258        return self.builds['variations']
259
260    def excludes(self, arch):
261        excludes = self.archs[arch]['excludes'].keys()
262        for exclude in self.archs[arch]['excludes']:
263            if 'all' not in self.archs[arch]['excludes'][exclude]:
264                excludes.remove(exclude)
265        return sorted(excludes)
266
267    def archs(self):
268        return sorted(self.archs.keys())
269
270    def arch_present(self, arch):
271        return arch in self.archs
272
273    def arch_bsps(self, arch):
274        return sorted(self.archs[arch]['bsps'])
275
276    def bsp_present(self, arch, bsp):
277        return bsp in self.archs[arch]['bsps']
278
279    def bsp_excludes(self, arch, bsp):
280        excludes = self.archs[arch]['excludes'].keys()
281        for exclude in self.archs[arch]['excludes']:
282            if bsp not in self.archs[arch]['excludes'][exclude]:
283                excludes.remove(exclude)
284        return sorted(excludes)
285
286    def bspopts(self, arch, bsp):
287        return self.archs[arch][bsp]['bspopts']
288
289    def base(self):
290        return self.builds['base']
291
292    def variant_options(self, variant):
293        if variant in self.builds['var_options']:
294            return self.builds['var_options'][variant]
295        return []
296
297    def profile_present(self, profile):
298        return profile in self.profiles
299
300    def profile_archs(self, profile):
301        return self.profiles[profile]['archs']
302
303    def profile_arch_bsps(self, profile, arch):
304        return self.profiles[profile]['bsps_%s' % (arch)]
305
306class build:
307
308    def __init__(self, config, version, prefix, tools, rtems, build_dir, options):
309        self.config = config
310        self.build_dir = build_dir
311        self.rtems_version = version
312        self.prefix = prefix
313        self.tools = tools
314        self.rtems = rtems
315        self.options = options
316        self.errors = { 'configure': 0,
317                        'build':     0,
318                        'tests':     0 }
319        self.counts = { 'h'        : 0,
320                        'exes'     : 0,
321                        'objs'     : 0,
322                        'libs'     : 0 }
323        self.warnings = warnings_counter(rtems)
324        self.results = results()
325        if not path.exists(path.join(rtems, 'configure')) or \
326           not path.exists(path.join(rtems, 'Makefile.in')) or \
327           not path.exists(path.join(rtems, 'cpukit')):
328            raise error.general('RTEMS source path does not look like RTEMS')
329
330    def _error_str(self):
331        return 'Status: configure:%d build:%d' % \
332            (self.errors['configure'], self.errors['build'])
333
334    def _path(self, arch, bsp):
335        return path.join(self.build_dir, arch, bsp)
336
337    def _archs(self, build_data):
338        return sorted(build_data.keys())
339
340    def _bsps(self, arch):
341        return self.config.arch_bsps(arch)
342
343    def _variations(self, arch, bsp):
344        def _match(var, vars):
345            matches = []
346            for v in vars:
347                if var in v.split('-'):
348                    matches += [v]
349            return matches
350
351        vars = self.config.variations()
352        for v in self.config.excludes(arch):
353            for m in _match(v, vars):
354                vars.remove(m)
355        for v in self.config.bsp_excludes(arch, bsp):
356            for m in _match(v, vars):
357                vars.remove(m)
358        return vars
359
360    def _arch_bsp_dir_make(self, arch, bsp):
361        if not path.exists(self._path(arch, bsp)):
362            path.mkdir(self._path(arch, bsp))
363
364    def _arch_bsp_dir_clean(self, arch, bsp):
365        if path.exists(self._path(arch, bsp)):
366            path.removeall(self._path(arch, bsp))
367
368    def _config_command(self, commands, arch, bsp):
369        cmd = [path.join(self.rtems, 'configure')]
370        commands += self.config.bspopts(arch, bsp)
371        for c in commands:
372            c = c.replace('@PREFIX@', self.prefix)
373            c = c.replace('@RTEMS_VERSION@', self.rtems_version)
374            c = c.replace('@ARCH@', arch)
375            c = c.replace('@BSP@', bsp)
376            cmd += [c]
377        return ' '.join(cmd)
378
379    def _build_set(self, variations):
380        build_set = { }
381        bs = self.config.base()
382        for var in variations:
383            build_set[var] = bs + self.config.variant_options(var)
384        return build_set
385
386    def _build_dir(self, arch, bsp, build):
387        return path.join(self._path(arch, bsp), build)
388
389    def _count_files(self, arch, bsp, build):
390        counts = { 'h'    : 0,
391                   'exes' : 0,
392                   'objs' : 0,
393                   'libs' : 0 }
394        for root, dirs, files in os.walk(self._build_dir(arch, bsp, build)):
395            for file in files:
396                if file.endswith('.exe'):
397                    counts['exes'] += 1
398                elif file.endswith('.o'):
399                    counts['objs'] += 1
400                elif file.endswith('.a'):
401                    counts['libs'] += 1
402                elif file.endswith('.h'):
403                    counts['h'] += 1
404        for f in self.counts:
405            if f in counts:
406                self.counts[f] = counts[f]
407        return counts
408
409    def build_arch_bsp(self, arch, bsp):
410        if not self.config.bsp_present(arch, bsp):
411            raise error.general('BSP not found: %s/%s' % (arch, bsp))
412        log.output('-' * 70)
413        log.notice('] BSP: %s/%s' % (arch, bsp))
414        log.notice('. Creating: %s' % (self._path(arch, bsp)))
415        self._arch_bsp_dir_clean(arch, bsp)
416        self._arch_bsp_dir_make(arch, bsp)
417        variations = self._variations(arch, bsp)
418        build_set = self._build_set(variations)
419        bsp_start = datetime.datetime.now()
420        bsp_warnings = warnings_counter(self.rtems)
421        env_path = os.environ['PATH']
422        os.environ['PATH'] = path.host(path.join(self.tools, 'bin')) + \
423                             os.pathsep + os.environ['PATH']
424        for bs in sorted(build_set.keys()):
425            warnings = warnings_counter(self.rtems)
426            start = datetime.datetime.now()
427            log.output('- ' * 35)
428            log.notice('. Configuring: %s' % (bs))
429            try:
430                result = '+ Pass'
431                bpath = self._build_dir(arch, bsp, bs)
432                path.mkdir(bpath)
433                config_cmd = self._config_command(build_set[bs], arch, bsp)
434                cmd = config_cmd
435                e = execute.capture_execution(log = warnings)
436                log.output('run: ' + cmd)
437                if self.options['dry-run']:
438                    exit_code = 0
439                else:
440                    exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath))
441                if exit_code != 0:
442                    result = '- FAIL'
443                    self.errors['configure'] += 1
444                    log.notice('- Configure failed: %s' % (bs))
445                    log.output('cmd failed: %s' % (cmd))
446                    if self.options['stop-on-error']:
447                        raise error.general('Configuring %s failed' % (bs))
448                else:
449                    log.notice('. Building: %s' % (bs))
450                    cmd = 'make'
451                    if 'jobs' in self.options:
452                        cmd += ' -j %s' % (self.options['jobs'])
453                    log.output('run: ' + cmd)
454                    if self.options['dry-run']:
455                        exit_code = 0
456                    else:
457                        exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath))
458                    if exit_code != 0:
459                        result = '- FAIL'
460                        self.errors['build'] += 1
461                        log.notice('- FAIL: %s: %s' % (bs, self._error_str()))
462                        log.output('cmd failed: %s' % (cmd))
463                        if self.options['stop-on-error']:
464                            raise error.general('Building %s failed' % (bs))
465                    files = self._count_files(arch, bsp, bs)
466                    log.notice('%s: %s: warnings:%d  exes:%d  objs:%s  libs:%d' % \
467                               (result, bs, warnings.get(),
468                                files['exes'], files['objs'], files['libs']))
469                log.notice('  %s' % (self._error_str()))
470                self.results.add(result[0] == '+', arch, bsp, config_cmd, warnings.get())
471            finally:
472                end = datetime.datetime.now()
473                if not self.options['no-clean']:
474                    log.notice('. Cleaning: %s' % (self._build_dir(arch, bsp, bs)))
475                    path.removeall(self._build_dir(arch, bsp, bs))
476            log.notice('^ Time %s' % (str(end - start)))
477            log.output('Warnings Report:')
478            log.output(warnings.report())
479            warnings.accumulate(bsp_warnings)
480            warnings.accumulate(self.warnings)
481        bsp_end = datetime.datetime.now()
482        log.notice('^ BSP Time %s' % (str(bsp_end - bsp_start)))
483        log.output('BSP Warnings Report:')
484        log.output(bsp_warnings.report())
485        os.environ['PATH'] = env_path
486
487    def build_arch(self, arch):
488        start = datetime.datetime.now()
489        log.output('=' * 70)
490        log.notice(']] Architecture: %s' % (arch))
491        if not self.config.arch_present(arch):
492            raise error.general('Architecture not found: %s' % (arch))
493        for bsp in self._bsps(arch):
494            self.build_arch_bsp(arch, bsp)
495        end = datetime.datetime.now()
496        log.notice('^ Architecture Time %s' % (str(end - start)))
497        log.notice('  warnings:%d  exes:%d  objs:%d  libs:%d' % \
498                   (self.warnings.get(), self.counts['exes'],
499                    self.counts['objs'], self.counts['libs']))
500        log.output('Architecture Warnings:')
501        log.output(self.warnings.report())
502
503    def build(self):
504        for arch in self.config.archs():
505            self.build_arch(arch)
506        log.notice('^ Profile Time %s' % (str(end - start)))
507        log.notice('+  warnings:%d  exes:%d  objs:%d  libs:%d' % \
508                   (self.warnings.get(), self.counts['exes'],
509                    self.counts['objs'], self.counts['libs']))
510        log.output('Profile Warnings:')
511        log.output(self.warnings.report())
512
513    def build_profile(self, profile):
514        if not self.config.profile_present(profile):
515            raise error.general('Profile not found: %s' % (profile))
516        start = datetime.datetime.now()
517        log.notice(']] Profile: %s' % (profile))
518        for arch in self.config.profile_archs(profile):
519            for bsp in self.config.profile_arch_bsps(profile, arch):
520                self.build_arch_bsp(arch, bsp)
521        end = datetime.datetime.now()
522        log.notice('^ Profile Time %s' % (str(end - start)))
523        log.notice('  warnings:%d  exes:%d  objs:%d  libs:%d' % \
524                   (self.warnings.get(), self.counts['exes'],
525                    self.counts['objs'], self.counts['libs']))
526        log.output('Profile Warnings:')
527        log.output(self.warnings.report())
528
529def run_args(args):
530    b = None
531    ec = 0
532    try:
533        #
534        # On Windows MSYS2 prepends a path to itself to the environment
535        # path. This means the RTEMS specific automake is not found and which
536        # breaks the bootstrap. We need to remove the prepended path. Also
537        # remove any ACLOCAL paths from the environment.
538        #
539        if os.name == 'nt':
540            cspath = os.environ['PATH'].split(os.pathsep)
541            if 'msys' in cspath[0] and cspath[0].endswith('bin'):
542                os.environ['PATH'] = os.pathsep.join(cspath[1:])
543
544        top = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
545        prefix = '/opt/rtems/%s' % (rtems_version())
546        tools = prefix
547        build_dir = 'bsp-builds'
548        logf = 'bsp-build-%s.txt' % (datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))
549        config_file = path.join(top, 'share', 'rtems', 'tester', 'rtems', 'rtems-bsps.ini')
550        if not path.exists(config_file):
551            config_file = path.join(top, 'tester', 'rtems', 'rtems-bsps.ini')
552
553        argsp = argparse.ArgumentParser()
554        argsp.add_argument('--prefix', help = 'Prefix to build the BSP.', type = str)
555        argsp.add_argument('--rtems-tools', help = 'The RTEMS tools directory.', type = str)
556        argsp.add_argument('--rtems', help = 'The RTEMS source tree.', type = str)
557        argsp.add_argument('--build-path', help = 'Path to build in.', type = str)
558        argsp.add_argument('--log', help = 'Log file.', type = str)
559        argsp.add_argument('--stop-on-error', help = 'Stop on an error.',
560                           action = 'store_true')
561        argsp.add_argument('--no-clean', help = 'Do not clean the build output.',
562                           action = 'store_true')
563        argsp.add_argument('--profiles', help = 'Build the listed profiles.',
564                           type = str, default = 'tier-1')
565        argsp.add_argument('--build', help = 'Build variation.', type = str, default='all')
566        argsp.add_argument('--arch', help = 'Build the specific architecture.', type = str)
567        argsp.add_argument('--bsp', help = 'Build the specific BSP.', type = str)
568        argsp.add_argument('--dry-run', help = 'Do not run the actual builds.',
569                           action = 'store_true')
570
571        opts = argsp.parse_args(args[1:])
572        if opts.log is not None:
573            logf = opts.log
574        log.default = log.log([logf])
575        log.notice('RTEMS Tools Project - RTEMS Kernel BSP Builder, %s' % (version.str()))
576        if opts.rtems is None:
577            raise error.general('No RTEMS source provided on the command line')
578        if opts.prefix is not None:
579            prefix = path.shell(opts.prefix)
580        if opts.rtems_tools is not None:
581            tools = path.shell(opts.rtems_tools)
582        if opts.build_path is not None:
583            build_dir = path.shell(opts.build_path)
584        if opts.bsp is not None and opts.arch is None:
585            raise error.general('BSP provided but no architecture')
586
587        config = configuration()
588        config.load(config_file, opts.build)
589
590        options = { 'stop-on-error' : opts.stop_on_error,
591                    'no-clean'      : opts.no_clean,
592                    'dry-run'       : opts.dry_run,
593                    'jobs'          : 8 }
594
595        b = build(config, rtems_version(), prefix, tools,
596                  path.shell(opts.rtems), build_dir, options)
597        if opts.arch is not None:
598            if opts.bsp is not None:
599                b.build_arch_bsp(opts.arch, opts.bsp)
600            else:
601                b.build_arch(opts.arch)
602        else:
603            for profile in opts.profiles.split(','):
604                b.build_profile(profile.strip())
605
606    except error.general as gerr:
607        print(gerr)
608        print('BSP Build FAILED', file = sys.stderr)
609        ec = 1
610    except error.internal as ierr:
611        print(ierr)
612        print('BSP Build FAILED', file = sys.stderr)
613        ec = 1
614    except error.exit as eerr:
615        pass
616    except KeyboardInterrupt:
617        log.notice('abort: user terminated')
618        ec = 1
619    if b is not None:
620        b.results.report()
621    sys.exit(ec)
622
623def run():
624    run_args(sys.argv)
625
626if __name__ == "__main__":
627    run()
Note: See TracBrowser for help on using the repository browser.