source: rtems-tools/tester/rt/check.py @ 85e14e0

5
Last change on this file since 85e14e0 was 85e14e0, checked in by Chris Johns <chrisj@…>, on 05/10/17 at 11:09:02

rtems-bsp-builder: Fix excluding builds.

Excluding builds was not working. This patch fixes that and it
also correctly handles mixed builds.

  • Property mode set to 100755
File size: 47.1 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2016-2017 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# 1. Redistributions of source code must retain the above copyright notice,
12# this list of conditions and the following disclaimer.
13#
14# 2. Redistributions in binary form must reproduce the above copyright notice,
15# this list of conditions and the following disclaimer in the documentation
16# and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31from __future__ import print_function
32
33import argparse
34import copy
35import datetime
36import operator
37import os
38import re
39import sys
40import textwrap
41
42import pprint
43
44try:
45    import configparser
46except:
47    import ConfigParser as configparser
48
49from rtemstoolkit import execute
50from rtemstoolkit import error
51from rtemstoolkit import host
52from rtemstoolkit import log
53from rtemstoolkit import path
54from rtemstoolkit import textbox
55from rtemstoolkit import version
56
57def rtems_version():
58    return version.version()
59
60def wrap(line, lineend = '', indent = 0, width = 75):
61    if type(line) is tuple or type(line) is list:
62        if len(line) >= 2:
63            s1 = line[0]
64        else:
65            s1 = ''
66        s2 = line[1:]
67    elif type(line) is str:
68        s1 = ''
69        s2 = [line]
70    else:
71        raise error.internal('line is not a tuple, list or string')
72    s1len = len(s1)
73    s = ''
74    first = True
75    for ss in s2:
76        if type(ss) is not str and type(ss) is not unicode:
77            raise error.internal('text needs to be a string')
78        for l in textwrap.wrap(ss, width = width - s1len - indent - 1):
79            s += '%s%s%s%s%s' % (' ' * indent, s1, l, lineend, os.linesep)
80            if first and s1len > 0:
81                s1 = ' ' * s1len
82    if lineend != '':
83        s = s[:0 - len(os.linesep) - 1] + os.linesep
84    return s
85
86def title():
87    return 'RTEMS Tools Project - RTEMS Kernel BSP Builder, %s' % (version.str())
88
89def command_line():
90    return wrap(('command: ', ' '.join(sys.argv)), lineend = '\\')
91
92class warnings_errors:
93
94    def __init__(self, rtems):
95        self.rtems = path.host(rtems)
96        self.reset()
97        self.groups = { 'groups'  : ['Shared', 'BSP', 'Network', 'Tests',
98                                     'LibCPU', 'CPU Kit'],
99                        'exclude' : '.*Makefile.*',
100                        'CPU Kit' : '.*cpukit/.*',
101                        'Network' : '.*libnetworking/.*|.*librpc/.*',
102                        'Tests'   : '.*testsuites/.*',
103                        'BSP'     : '.*libbsp/.*',
104                        'LibCPU'  : '.*libcpu/.*',
105                        'Shared'  : '.*shared/.*' }
106        self.arch = None
107        self.bsp = None
108        self.build = None
109
110    def _opts(self, arch = None, bsp = None, build = None):
111        if arch is None:
112            arch = self.arch
113        if bsp is None:
114            bsp = self.bsp
115        if build is None:
116            build = self.build
117        return arch, bsp, build
118
119    def _key(self, arch, bsp, build):
120        arch, bsp, build = self._opts(arch, bsp, build)
121        return '%s/%s-%s' % (arch, bsp, build)
122
123    def _get_warnings(self, arch = None, bsp = None, build = None):
124        arch, bsp, build = self._opts(arch = arch, bsp = bsp, build = build)
125        if arch is None:
126            arch = '.*'
127        if bsp is None:
128            bsp = '.*'
129        if build is None:
130            build = '.*'
131        selector = re.compile('^%s/%s-%s$' % (arch, bsp, build))
132        warnings = [w for w in self.warnings if selector.match(w)]
133        return sorted(warnings)
134
135    def _total(self, archive):
136        total = 0
137        for a in archive:
138            total += archive[a]
139        return total
140
141    def _analyze(self, warnings, exclude):
142        def _group(data, category, name, warning, count, groups, group_regx):
143            if 'groups' not in data:
144                data['groups'] = { }
145            if category not in data['groups']:
146                data['groups'][category] = { 'totals' : { } }
147            if name not in data['groups'][category]:
148                data['groups'][category][name] = { }
149            for group in groups:
150                if group not in data['groups'][category]['totals']:
151                    data['groups'][category]['totals'][group] = 0
152                if group not in data['groups'][category][name]:
153                    data['groups'][category][name][group] = 0
154                if group_regx[group].match(warning):
155                    data['groups'][category][name][group] += count
156                    data['groups'][category]['totals'][group] += count
157                    break
158
159        def _update(data, category, name, warning, count, groups, group_regx):
160            if category not in data:
161                data[category] = { }
162            if name not in data[category]:
163                data[category][name] = { }
164            if warning not in data[category][name]:
165                data[category][name][warning] = 0
166            data[category][name][warning] += count
167            _group(data, category, name,  w, count, groups, group_regx)
168
169        data = { }
170        group_regx = { }
171        for group in self.groups['groups']:
172            group_regx[group] = re.compile(self.groups[group])
173        exclude_regx = re.compile(exclude)
174        for warning in warnings:
175            arch = warning.split('/', 1)[0]
176            arch_bsp = warning.split('-', 1)[0]
177            build = warning.split('-', 1)[1]
178            for w in self.warnings[warning]:
179                if not exclude_regx.match(w):
180                    count = self.warnings[warning][w]
181                    _update(data, 'arch',     arch,     w, count,
182                           self.groups['groups'], group_regx)
183                    _update(data, 'arch_bsp', arch_bsp, w, count,
184                           self.groups['groups'], group_regx)
185                    _update(data, 'build',  build,  w, count,
186                           self.groups['groups'], group_regx)
187        for category in ['arch', 'arch_bsp', 'build']:
188            common = {}
189            for name in data[category]:
190                for w in data[category][name]:
191                    if w not in common:
192                        for other in [n for n in data[category] if n != name]:
193                            if w in data[category][other]:
194                                common[w] = data[category][name][w]
195                                _group(data, category, 'common', w, common[w],
196                                       self.groups['groups'], group_regx)
197            data[category]['common'] = common
198        return data
199
200    def _report_category(self, label, warnings, group_counts):
201        width = 70
202        cols_1 = [width]
203        cols_2 = [8, width - 8]
204        cols_4 = textbox.even_columns(4, width)
205        cols_2_4 = textbox.merge_columns([cols_2, cols_4])
206        s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
207        s += textbox.row(cols_1, [' ' + label], indent = 1)
208        s += textbox.line(cols_1, marker = '+', indent = 1)
209        builds = ['common'] + sorted([b for b in warnings if b != 'common'])
210        common = warnings['common']
211        for build in builds:
212            build_warnings = warnings[build]
213            if build is not 'common':
214                build_warnings = [w for w in build_warnings if w not in common]
215            s += textbox.row(cols_1,
216                             [' %s : %d warning(s)' % (build,
217                                                       len(build_warnings))],
218                             indent = 1)
219            if len(build_warnings) == 0:
220                s += textbox.line(cols_1, marker = '+', indent = 1)
221            else:
222                s += textbox.line(cols_4, marker = '+', indent = 1)
223                if build not in group_counts:
224                    gs = [0 for group in self.groups['groups']]
225                else:
226                    gs = []
227                    for g in range(0, len(self.groups['groups'])):
228                        group = self.groups['groups'][g]
229                        if group in group_counts[build]:
230                            count = group_counts[build][group]
231                        else:
232                            count = 0
233                        gs += ['%*s' % (cols_4[g % 4] - 2,
234                                        '%s : %4d' % (group, count))]
235                    for row in range(0, len(self.groups['groups']), 4):
236                        if row + 4 > len(self.groups['groups']):
237                            d = gs[row:] + \
238                                ['' for r in range(row,
239                                                   len(self.groups['groups']))]
240                        else:
241                            d = gs[row:+4]
242                        s += textbox.row(cols_4, d, indent = 1)
243                s += textbox.line(cols_2_4, marker = '+', indent = 1)
244                vw = sorted([(w, warnings[build][w]) for w in build_warnings],
245                            key = operator.itemgetter(1),
246                            reverse = True)
247                for w in vw:
248                    c1 = '%6d' % w[1]
249                    for l in textwrap.wrap(' ' + w[0], width = cols_2[1] - 3):
250                        s += textbox.row(cols_2, [c1, l], indent = 1)
251                        c1 = ' ' * 6
252                s += textbox.line(cols_2, marker = '+', indent = 1)
253        return s
254
255    def _report_warning_map(self):
256        builds = self.messages['warnings']
257        width = 70
258        cols_1 = [width]
259        s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
260        s += textbox.row(cols_1, [' Warning Map'], indent = 1)
261        s += textbox.line(cols_1, marker = '+', indent = 1)
262        for build in builds:
263            messages = builds[build]
264            s += textbox.row(cols_1, [' %s : %d' % (build, len(messages))], indent = 1)
265            s += textbox.line(cols_1, marker = '+', indent = 1)
266            for msg in messages:
267                for l in textwrap.wrap(msg, width = width - 3):
268                    s += textbox.row(cols_1, [' ' + l], indent = 1)
269                for l in textwrap.wrap(messages[msg], width = width - 3 - 4):
270                    s += textbox.row(cols_1, ['    ' + l], indent = 1)
271            s += textbox.line(cols_1, marker = '+', indent = 1)
272        return s
273
274    def report(self):
275        arch, bsp, build = self._opts()
276        warnings = self._get_warnings(arch, bsp, build)
277        total = 0
278        for build in warnings:
279            total += self._total(self.warnings[build])
280        if total == 0:
281            s = ' No warnings'
282        else:
283            data = self._analyze(warnings, self.groups['exclude'])
284            s = self._report_category('By Architecture (total : %d)' % (total),
285                                      data['arch'], data['groups']['arch'])
286            s += os.linesep
287            s += self._report_category('By BSP (total : %d)' % (total),
288                                       data['arch_bsp'], data['groups']['arch_bsp'])
289            s += os.linesep
290            s += self._report_category('By Build (total : %d)' % (total),
291                                       data['build'], data['groups']['build'])
292            s += os.linesep
293            s += self._report_warning_map()
294            s += os.linesep
295
296        return s
297
298    def set_build(self, arch, bsp, build):
299        self.arch = arch
300        self.bsp = bsp
301        self.build = build
302        self.build_key = '%s/%s-%s' % (arch, bsp, build)
303        if self.build_key not in self.warnings:
304            self.warnings[self.build_key] = {}
305        if self.build_key not in self.errors:
306            self.errors[self.build_key] = {}
307
308    def clear_build(self):
309        self.arch = None
310        self.bsp = None
311        self.build = None
312        self.build_key = None
313
314    def get_warning_count(self):
315        return self.warning_count
316
317    def get_error_count(self):
318        return self.error_count
319
320    def reset(self):
321        self.warnings = { }
322        self.warning_count = 0
323        self.errors = { }
324        self.error_count = 0
325        self.messages = { 'warnings' : { }, 'errors' : { } }
326
327    def get_warning_messages(self, arch = None, bsp = None, build = None):
328        key = self._key(arch, bsp, build)
329        if key not in self.messages['warnings']:
330            return []
331        messages = self.messages['warnings'][key]
332        return ['%s %s' % (m, messages[m]) for m in messages]
333
334    def get_error_messages(self, arch = None, bsp = None, build = None):
335        key = self._key(arch, bsp, build)
336        if key not in self.messages['errors']:
337            return []
338        messages = self.messages['errors'][key]
339        return ['%s %s' % (m, messages[m]) for m in messages]
340
341    def output(self, text):
342        def _line_split(line, source_base):
343            ls = line.split(' ', 1)
344            fname = ls[0].split(':')
345            #
346            # Ignore compiler option warnings.
347            #
348            if len(fname) < 4:
349                return None
350            p = path.abspath(fname[0])
351            p = p.replace(source_base, '')
352            if path.isabspath(p):
353                p = p[1:]
354            return p, fname[1], fname[2], ls[1]
355
356        if self.build_key is not None and \
357           (' warning:' in text or ' error:' in text):
358            for l in text.splitlines():
359                if ' warning:' in l:
360                    self.warning_count += 1
361                    archive = self.warnings[self.build_key]
362                    messages = 'warnings'
363                elif ' error:' in l:
364                    self.error_count += 1
365                    archive = self.errors[self.build_key]
366                    messages = 'errors'
367                else:
368                    continue
369                line_parts = _line_split(l, self.rtems)
370                if line_parts is not None:
371                    src, line, pos, msg = line_parts
372                    where = '%s:%s:%s' % (src, line, pos)
373                    if where not in archive:
374                        archive[where] = 1
375                    else:
376                        archive[where] += 1
377                    if self.build_key not in self.messages[messages]:
378                        self.messages[messages][self.build_key] = { }
379                    self.messages[messages][self.build_key][where] = msg
380
381        log.output(text)
382
383class results:
384
385    def __init__(self):
386        self.passes = []
387        self.fails = []
388
389    def _arch_bsp(self, arch, bsp):
390        return '%s/%s' % (arch, bsp)
391
392    def add(self, good, arch, bsp, configure, warnings, error_messages):
393        if good:
394            self.passes += [(arch, bsp, configure, warnings, None)]
395        else:
396            self.fails += [(arch, bsp, configure, warnings, error_messages)]
397
398    def report(self):
399        log.notice('* Passes: %d   Failures: %d' %
400                   (len(self.passes), len(self.fails)))
401        log.output()
402        log.output('Build Report')
403        log.output('   Passes: %d   Failures: %d' %
404                   (len(self.passes), len(self.fails)))
405        log.output(' Failures:')
406        if len(self.fails) == 0:
407            log.output('  None')
408        else:
409            max_col = 0
410            for f in self.fails:
411                arch_bsp = self._arch_bsp(f[0], f[1])
412                if len(arch_bsp) > max_col:
413                    max_col = len(arch_bsp)
414            for f in self.fails:
415                config_cmd = f[2]
416                config_at = config_cmd.find('configure')
417                if config_at != -1:
418                    config_cmd = config_cmd[config_at:]
419                log.output(' %*s:' % (max_col + 2, self._arch_bsp(f[0], f[1])))
420                s1 = ' ' * 6
421                log.output(wrap([s1, config_cmd], lineend = '\\', width = 75))
422                if f[4] is not None:
423                    s1 = ' ' * len(s1)
424                    for msg in f[4]:
425                        log.output(wrap([s1, msg], lineend = '\\'))
426        log.output(' Passes:')
427        if len(self.passes) == 0:
428            log.output('  None')
429        else:
430            max_col = 0
431            for f in self.passes:
432                arch_bsp = self._arch_bsp(f[0], f[1])
433                if len(arch_bsp) > max_col:
434                    max_col = len(arch_bsp)
435            for f in self.passes:
436                config_cmd = f[2]
437                config_at = config_cmd.find('configure')
438                if config_at != -1:
439                    config_cmd = config_cmd[config_at:]
440                log.output(' %s (%5d):' % (self._arch_bsp(f[0], f[1]), f[3]))
441                log.output(wrap([' ' * 6, config_cmd], lineend = '\\', width = 75))
442
443class configuration:
444
445    def __init__(self):
446        self.config = configparser.ConfigParser()
447        self.name = None
448        self.archs = { }
449        self.builds_ = { }
450        self.profiles = { }
451        self.configurations = { }
452
453    def __str__(self):
454        import pprint
455        s = self.name + os.linesep
456        s += 'Archs:' + os.linesep + \
457             pprint.pformat(self.archs, indent = 1, width = 80) + os.linesep
458        s += 'Builds:' + os.linesep + \
459             pprint.pformat(self.builds_, indent = 1, width = 80) + os.linesep
460        s += 'Profiles:' + os.linesep + \
461             pprint.pformat(self.profiles, indent = 1, width = 80) + os.linesep
462        return s
463
464    def _get_item(self, section, label, err = True):
465        try:
466            rec = self.config.get(section, label).replace(os.linesep, ' ')
467            return rec
468        except:
469            if err:
470                raise error.general('config: no "%s" found in "%s"' % (label, section))
471        return None
472
473    def _get_items(self, section, err = True):
474        try:
475            items = [(name, key.replace(os.linesep, ' ')) \
476                     for name, key in self.config.items(section)]
477            return items
478        except:
479            if err:
480                raise error.general('config: section "%s" not found' % (section))
481        return []
482
483    def _comma_list(self, section, label, error = True):
484        items = self._get_item(section, label, error)
485        if items is None:
486            return []
487        return sorted(set([a.strip() for a in items.split(',')]))
488
489    def _get_item_names(self, section, err = True):
490        try:
491            return [item[0] for item in self.config.items(section)]
492        except:
493            if err:
494                raise error.general('config: section "%s" not found' % (section))
495        return []
496
497    def _build_options(self, build, nesting = 0):
498        if ':' in build:
499            section, name = build.split(':', 1)
500            opts = [self._get_item(section, name)]
501            return opts
502        builds = self.builds_['builds']
503        if build not in builds:
504            raise error.general('build %s not found' % (build))
505        if nesting > 20:
506            raise error.general('nesting build %s' % (build))
507        options = []
508        for option in self.builds_['builds'][build]:
509            if ':' in option:
510                section, name = option.split(':', 1)
511                opts = [self._get_item(section, name)]
512            else:
513                opts = self._options(option, nesting + 1)
514            for opt in opts:
515                if opt not in options:
516                    options += [opt]
517        return options
518
519    def load(self, name, build):
520        if not path.exists(name):
521            raise error.general('config: cannot read configuration: %s' % (name))
522        self.name = name
523        try:
524            self.config.read(name)
525        except configparser.ParsingError as ce:
526            raise error.general('config: %s' % (ce))
527        archs = []
528        self.profiles['profiles'] = self._comma_list('profiles', 'profiles', error = False)
529        if len(self.profiles['profiles']) == 0:
530            self.profiles['profiles'] = ['tier-%d' % (t) for t in range(1,4)]
531        for p in self.profiles['profiles']:
532            profile = {}
533            profile['name'] = p
534            profile['archs'] = self._comma_list(profile['name'], 'archs')
535            archs += profile['archs']
536            for arch in profile['archs']:
537                bsps = 'bsps_%s' % (arch)
538                profile[bsps] = self._comma_list(profile['name'], bsps)
539            self.profiles[profile['name']] = profile
540        for a in set(archs):
541            arch = {}
542            arch['excludes'] = {}
543            for exclude in self._comma_list(a, 'exclude', error = False):
544                arch['excludes'][exclude] = ['all']
545            for i in self._get_items(a, False):
546                if i[0].startswith('exclude-'):
547                    exclude = i[0][len('exclude-'):]
548                    if exclude not in arch['excludes']:
549                        arch['excludes'][exclude] = []
550                    arch['excludes'][exclude] += sorted(set([b.strip() for b in i[1].split(',')]))
551            arch['bsps'] = self._comma_list(a, 'bsps', error = False)
552            for b in arch['bsps']:
553                arch[b] = {}
554                arch[b]['bspopts'] = self._comma_list(a, 'bspopts_%s' % (b), error = False)
555            self.archs[a] = arch
556        builds = {}
557        builds['default'] = self._get_item('builds', 'default')
558        if build is None:
559            build = builds['default']
560        builds['config'] = { }
561        for config in self._get_items('config'):
562            builds['config'][config[0]] = config[1]
563        builds['build'] = build
564        builds_ = self._get_item_names('builds')
565        builds['builds'] = {}
566        for build in builds_:
567            build_builds = self._comma_list('builds', build)
568            has_config = False
569            has_build = False
570            for b in build_builds:
571                if ':' in b:
572                    if has_build:
573                        raise error.general('config and build in build: %s' % (build))
574                    has_config = True
575                else:
576                    if has_config:
577                        raise error.general('config and build in build: %s' % (build))
578                    has_build = True
579            builds['builds'][build] = build_builds
580        self.builds_ = builds
581
582    def build(self):
583        return self.builds_['build']
584
585    def builds(self):
586        if self.builds_['build'] in self.builds_['builds']:
587            build = copy.copy(self.builds_['builds'][self.builds_['build']])
588            if ':' in build[0]:
589                return [self.builds_['build']]
590            return build
591        return None
592
593    def build_options(self, build):
594        return ' '.join(self._build_options(build))
595
596    def excludes(self, arch):
597        excludes = self.archs[arch]['excludes'].keys()
598        for exclude in self.archs[arch]['excludes']:
599            if 'all' not in self.archs[arch]['excludes'][exclude]:
600                excludes.remove(exclude)
601        return sorted(excludes)
602
603    def archs(self):
604        return sorted(self.archs.keys())
605
606    def arch_present(self, arch):
607        return arch in self.archs
608
609    def arch_bsps(self, arch):
610        return sorted(self.archs[arch]['bsps'])
611
612    def bsp_present(self, arch, bsp):
613        return bsp in self.archs[arch]['bsps']
614
615    def bsp_excludes(self, arch, bsp):
616        excludes = self.archs[arch]['excludes'].keys()
617        for exclude in self.archs[arch]['excludes']:
618            if 'all' not in self.archs[arch]['excludes'][exclude] and \
619               bsp not in self.archs[arch]['excludes'][exclude]:
620                excludes.remove(exclude)
621        return sorted(excludes)
622
623    def bspopts(self, arch, bsp):
624        return self.archs[arch][bsp]['bspopts']
625
626    def profile_present(self, profile):
627        return profile in self.profiles
628
629    def profile_archs(self, profile):
630        return self.profiles[profile]['archs']
631
632    def profile_arch_bsps(self, profile, arch):
633        return self.profiles[profile]['bsps_%s' % (arch)]
634
635    def report(self, profiles = True, builds = True, architectures = True):
636        width = 70
637        cols_1 = [width]
638        cols_2 = [10, width - 10]
639        s = textbox.line(cols_1, line = '=', marker = '+', indent = 1)
640        s1 = ' File'
641        colon = ':'
642        for l in textwrap.wrap(self.name, width = cols_2[1] - 3):
643            s += textbox.row(cols_2, [s1, ' ' + l], marker = colon, indent = 1)
644            colon = ' '
645            s1 = ' ' * len(s1)
646        s += textbox.line(cols_1, marker = '+', indent = 1)
647        s += os.linesep
648        if profiles:
649            s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
650            profiles = sorted(self.profiles['profiles'])
651            bsps = 0
652            for profile in profiles:
653                archs = sorted(self.profiles[profile]['archs'])
654                for arch in archs:
655                    bsps += len(self.profiles[profile]['bsps_%s' % (arch)])
656            s += textbox.row(cols_1,
657                             [' Profiles : %d/%d' % (len(archs), bsps)],
658                             indent = 1)
659            for profile in profiles:
660                textbox.row(cols_2,
661                            [profile, self.profiles[profile]['name']],
662                            indent = 1)
663            s += textbox.line(cols_1, marker = '+', indent = 1)
664            for profile in profiles:
665                s += textbox.row(cols_1, [' %s' % (profile)], indent = 1)
666                profile = self.profiles[profile]
667                archs = sorted(profile['archs'])
668                for arch in archs:
669                    s += textbox.line(cols_2, marker = '+', indent = 1)
670                    s1 = ' ' + arch
671                    for l in textwrap.wrap(', '.join(profile['bsps_%s' % (arch)]),
672                                           width = cols_2[1] - 2):
673                        s += textbox.row(cols_2, [s1, ' ' + l], indent = 1)
674                        s1 = ' ' * len(s1)
675                s += textbox.line(cols_2, marker = '+', indent = 1)
676            s += os.linesep
677        if builds:
678            s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
679            s += textbox.row(cols_1,
680                             [' Builds:  %s (default)' % (self.builds_['default'])],
681                             indent = 1)
682            builds = self.builds_['builds']
683            bsize = 0
684            for build in builds:
685                if len(build) > bsize:
686                    bsize = len(build)
687            cols_b = [bsize + 2, width - bsize - 2]
688            s += textbox.line(cols_b, marker = '+', indent = 1)
689            for build in builds:
690                s1 = ' ' + build
691                for l in textwrap.wrap(', '.join(builds[build]),
692                                       width = cols_b[1] - 2):
693                    s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
694                    s1 = ' ' * len(s1)
695                s += textbox.line(cols_b, marker = '+', indent = 1)
696            configs = self.builds_['config']
697            s += textbox.row(cols_1,
698                             [' Configure Options: %d' % (len(configs))],
699                             indent = 1)
700            csize = 0
701            for config in configs:
702                if len(config) > csize:
703                    csize = len(config)
704            cols_c = [csize + 3, width - csize - 3]
705            s += textbox.line(cols_c, marker = '+', indent = 1)
706            for config in configs:
707                s1 = ' ' + config
708                for l in textwrap.wrap(configs[config], width = cols_c[1] - 3):
709                    s += textbox.row(cols_c, [s1, ' ' + l], indent = 1)
710                    s1 = ' ' * len(s1)
711                s += textbox.line(cols_c, marker = '+', indent = 1)
712            s += os.linesep
713        if architectures:
714            s += textbox.line(cols_1, line = '=', marker = '+', indent = 1)
715            archs = sorted(self.archs.keys())
716            bsps = 0
717            asize = 0
718            for arch in archs:
719                if len(arch) > asize:
720                    asize = len(arch)
721                bsps += len(self.archs[arch]['bsps'])
722            s += textbox.row(cols_1,
723                             [' Architectures : %d (bsps: %d)' % (len(archs), bsps)],
724                             indent = 1)
725            cols_a = [asize + 2, width - asize - 2]
726            s += textbox.line(cols_a, marker = '+', indent = 1)
727            for arch in archs:
728                s += textbox.row(cols_a,
729                                 [' ' + arch, ' %d' % (len(self.archs[arch]['bsps']))],
730                                 indent = 1)
731            s += textbox.line(cols_a, marker = '+', indent = 1)
732            for archn in archs:
733                arch = self.archs[archn]
734                if len(arch['bsps']) > 0:
735                    bsize = 0
736                    for bsp in arch['bsps']:
737                        if len(bsp) > bsize:
738                            bsize = len(bsp)
739                    cols_b = [bsize + 3, width - bsize - 3]
740                    s += textbox.row(cols_1, [' ' + archn + ':'], indent = 1)
741                    s += textbox.line(cols_b, marker = '+', indent = 1)
742                    for bsp in arch['bsps']:
743                        s1 = ' ' + bsp
744                        bspopts = ' '.join(arch[bsp]['bspopts'])
745                        if len(bspopts):
746                            for l in textwrap.wrap('bopt: ' + bspopts,
747                                                   width = cols_b[1] - 3):
748                                s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
749                                s1 = ' ' * len(s1)
750                        excludes = []
751                        for exclude in arch['excludes']:
752                            if 'all' in arch['excludes'][exclude] or \
753                               bsp in arch['excludes'][exclude]:
754                                excludes += [exclude]
755                        excludes = ', '.join(excludes)
756                        if len(excludes):
757                            for l in textwrap.wrap('ex: ' + excludes,
758                                                   width = cols_b[1] - 3):
759                                s += textbox.row(cols_b, [s1, ' ' + l], indent = 1)
760                                s1 = ' ' * len(s1)
761                        if len(bspopts) == 0 and len(excludes) == 0:
762                            s += textbox.row(cols_b, [s1, ' '], indent = 1)
763                    s += textbox.line(cols_b, marker = '+', indent = 1)
764            s += os.linesep
765        return s
766
767class build:
768
769    def __init__(self, config, version, prefix, tools, rtems, build_dir, options):
770        self.config = config
771        self.build_dir = build_dir
772        self.rtems_version = version
773        self.prefix = prefix
774        self.tools = tools
775        self.rtems = rtems
776        self.options = options
777        self.errors = { 'configure': 0,
778                        'build':     0,
779                        'tests':     0,
780                        'fails':     []}
781        self.counts = { 'h'        : 0,
782                        'exes'     : 0,
783                        'objs'     : 0,
784                        'libs'     : 0 }
785        self.warnings_errors = warnings_errors(rtems)
786        self.results = results()
787        if not path.exists(path.join(rtems, 'configure')) or \
788           not path.exists(path.join(rtems, 'Makefile.in')) or \
789           not path.exists(path.join(rtems, 'cpukit')):
790            raise error.general('RTEMS source path does not look like RTEMS')
791
792    def _error_str(self):
793        return 'Status: configure:%d build:%d' % \
794            (self.errors['configure'], self.errors['build'])
795
796    def _path(self, arch, bsp):
797        return path.join(self.build_dir, arch, bsp)
798
799    def _archs(self, build_data):
800        return sorted(build_data.keys())
801
802    def _bsps(self, arch):
803        return self.config.arch_bsps(arch)
804
805    def _build(self):
806        return self.config.build()
807
808    def _builds(self, arch, bsp):
809        builds = self.config.builds()
810        if builds is None:
811            return None
812        excludes = set(self.config.excludes(arch) +
813                       self.config.bsp_excludes(arch, bsp))
814        remove = []
815        for e in excludes:
816            remove += [b for b in builds if e in b]
817        for b in remove:
818            builds.remove(b)
819        return builds
820
821    def _arch_bsp_dir_make(self, arch, bsp):
822        if not path.exists(self._path(arch, bsp)):
823            path.mkdir(self._path(arch, bsp))
824
825    def _arch_bsp_dir_clean(self, arch, bsp):
826        if path.exists(self._path(arch, bsp)):
827            path.removeall(self._path(arch, bsp))
828
829    def _config_command(self, commands, arch, bsp):
830        if type(commands) is not list:
831            commands = [commands]
832        cmd = [path.join(self.rtems, 'configure')]
833        commands += self.config.bspopts(arch, bsp)
834        for c in commands:
835            c = c.replace('@PREFIX@', self.prefix)
836            c = c.replace('@RTEMS_VERSION@', self.rtems_version)
837            c = c.replace('@ARCH@', arch)
838            c = c.replace('@BSP@', bsp)
839            cmd += [c]
840        return ' '.join(cmd)
841
842    def _build_set(self, builds):
843        build_set = { }
844        for build in builds:
845            build_set[build] = self.config.build_options(build)
846        return build_set
847
848    def _build_dir(self, arch, bsp, build):
849        return path.join(self._path(arch, bsp), build)
850
851    def _count_files(self, arch, bsp, build):
852        counts = { 'h'    : 0,
853                   'exes' : 0,
854                   'objs' : 0,
855                   'libs' : 0 }
856        for root, dirs, files in os.walk(self._build_dir(arch, bsp, build)):
857            for file in files:
858                if file.endswith('.exe'):
859                    counts['exes'] += 1
860                elif file.endswith('.o'):
861                    counts['objs'] += 1
862                elif file.endswith('.a'):
863                    counts['libs'] += 1
864                elif file.endswith('.h'):
865                    counts['h'] += 1
866        for f in self.counts:
867            if f in counts:
868                self.counts[f] += counts[f]
869        return counts
870
871    def _have_failures(self, fails):
872        return len(fails) != 0
873
874    def _warnings_report(self):
875        if self.options['warnings-report'] is not None:
876            with open(self.options['warnings-report'], 'w') as f:
877                f.write(title() + os.linesep)
878                f.write(os.linesep)
879                f.write('Date: %s%s' % (datetime.datetime.now().strftime('%c'),
880                                        os.linesep))
881                f.write(os.linesep)
882                f.write(command_line() + os.linesep)
883                f.write(self.warnings_errors.report())
884
885    def _finished(self):
886        log.notice('+  warnings:%d  exes:%d  objs:%d  libs:%d' % \
887                   (self.warnings_errors.get_warning_count(), self.counts['exes'],
888                    self.counts['objs'], self.counts['libs']))
889        log.output()
890        log.output('Warnings:')
891        log.output(self.warnings_errors.report())
892        log.output()
893        log.notice('Failures:')
894        log.notice(self.failures_report(self.errors['fails']))
895        self._warnings_report()
896
897    def failures_report(self, fails):
898        if not self._have_failures(fails):
899            return ' No failure(s)'
900        absize = 0
901        bsize = 0
902        ssize = 0
903        for f in fails:
904            arch_bsp = '%s/%s' % (f[1], f[2])
905            if len(arch_bsp) > absize:
906                absize = len(arch_bsp)
907            if len(f[3]) > bsize:
908                bsize = len(f[3])
909            if len(f[0]) > ssize:
910                ssize = len(f[0])
911        fc = 1
912        s = ''
913        for f in fails:
914            fcl = ' %3d' % (fc)
915            arch_bsp = '%s/%s' % (f[1], f[2])
916            state = f[0]
917            s += '%s %-*s %-*s %-*s:%s' % \
918                 (fcl, bsize, f[3], absize, arch_bsp, ssize, state, os.linesep)
919            s1 = ' ' * 6
920            s += wrap((s1, 'configure: ' + f[4]), lineend = '\\', width = 75)
921            for e in self.warnings_errors.get_error_messages(f[1], f[2], f[3]):
922                s += wrap([s1, 'error: ' + e])
923            fc += 1
924        return s
925
926    def build_arch_bsp(self, arch, bsp):
927        if not self.config.bsp_present(arch, bsp):
928            raise error.general('BSP not found: %s/%s' % (arch, bsp))
929        log.output('-' * 70)
930        log.notice('] BSP: %s/%s' % (arch, bsp))
931        log.notice('. Creating: %s' % (self._path(arch, bsp)))
932        self._arch_bsp_dir_clean(arch, bsp)
933        self._arch_bsp_dir_make(arch, bsp)
934        builds = self._builds(arch, bsp)
935        if builds is None:
936            raise error.general('build not found: %s' % (self._build()))
937        build_set = self._build_set(builds)
938        bsp_start = datetime.datetime.now()
939        env_path = os.environ['PATH']
940        os.environ['PATH'] = path.host(path.join(self.tools, 'bin')) + \
941                             os.pathsep + os.environ['PATH']
942        fails = []
943        for bs in sorted(build_set.keys()):
944            self.warnings_errors.set_build(arch, bsp, bs)
945            start = datetime.datetime.now()
946            log.output('- ' * 35)
947            log.notice('. Configuring: %s' % (bs))
948            try:
949                warnings = self.warnings_errors.get_warning_count()
950                result = '+ Pass'
951                bpath = self._build_dir(arch, bsp, bs)
952                good = True
953                error_messages = None
954                path.mkdir(bpath)
955                config_cmd = self._config_command(build_set[bs], arch, bsp)
956                cmd = config_cmd
957                e = execute.capture_execution(log = self.warnings_errors)
958                log.output(wrap(('run: ', cmd), lineend = '\\'))
959                if self.options['dry-run']:
960                    exit_code = 0
961                else:
962                    exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath))
963                if exit_code != 0:
964                    result = '- FAIL'
965                    failure = ('configure', arch, bsp, bs, config_cmd)
966                    fails += [failure]
967                    self.errors['configure'] += 1
968                    self.errors['fails'] += [failure]
969                    log.notice('- Configure failed: %s' % (bs))
970                    log.output('cmd failed: %s' % (cmd))
971                    good = False
972                else:
973                    log.notice('. Building: %s' % (bs))
974                    cmd = 'make'
975                    if 'jobs' in self.options:
976                        cmd += ' -j %s' % (self.options['jobs'])
977                    log.output('run: ' + cmd)
978                    if self.options['dry-run']:
979                        exit_code = 0
980                    else:
981                        exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath))
982                    if exit_code != 0:
983                        error_messages = self.warnings_errors.get_error_messages()
984                        result = '- FAIL'
985                        failure = ('build', arch, bsp, bs, config_cmd, error_messages)
986                        fails += [failure]
987                        self.errors['build'] += 1
988                        self.errors['fails'] += [failure]
989                        log.notice('- FAIL: %s: %s' % (bs, self._error_str()))
990                        log.output('cmd failed: %s' % (cmd))
991                        good = False
992                    files = self._count_files(arch, bsp, bs)
993                    log.notice('%s: %s: warnings:%d  exes:%d  objs:%s  libs:%d' % \
994                               (result, bs,
995                                self.warnings_errors.get_warning_count() - warnings,
996                                files['exes'], files['objs'], files['libs']))
997                log.notice('  %s' % (self._error_str()))
998                self.results.add(good, arch, bsp, config_cmd,
999                                 self.warnings_errors.get_warning_count() - warnings,
1000                                 error_messages)
1001                if not good and self.options['stop-on-error']:
1002                    raise error.general('Configuring %s failed' % (bs))
1003            finally:
1004                end = datetime.datetime.now()
1005                if not self.options['no-clean']:
1006                    log.notice('. Cleaning: %s' % (self._build_dir(arch, bsp, bs)))
1007                    path.removeall(self._build_dir(arch, bsp, bs))
1008            log.notice('^ Time %s' % (str(end - start)))
1009            self.warnings_errors.clear_build()
1010        bsp_end = datetime.datetime.now()
1011        log.notice('^ BSP Time %s' % (str(bsp_end - bsp_start)))
1012        log.output('Failure Report:')
1013        log.output(self.failures_report(fails))
1014        os.environ['PATH'] = env_path
1015
1016    def build_bsp(self, arch, bsp):
1017        self.build_arch_bsp(arch, bsp)
1018        self._finished()
1019
1020    def build_arch(self, arch):
1021        start = datetime.datetime.now()
1022        log.output('=' * 70)
1023        log.notice(']] Architecture: %s' % (arch))
1024        if not self.config.arch_present(arch):
1025            raise error.general('Architecture not found: %s' % (arch))
1026        for bsp in self._bsps(arch):
1027            self.build_arch_bsp(arch, bsp)
1028        end = datetime.datetime.now()
1029        log.notice('^ Architecture Time %s' % (str(end - start)))
1030        self._finished()
1031
1032    def build(self):
1033        for arch in self.config.archs():
1034            self.build_arch(arch)
1035        log.notice('^ Profile Time %s' % (str(end - start)))
1036        self._finished()
1037
1038    def build_profile(self, profile):
1039        if not self.config.profile_present(profile):
1040            raise error.general('Profile not found: %s' % (profile))
1041        start = datetime.datetime.now()
1042        log.notice(']] Profile: %s' % (profile))
1043        for arch in self.config.profile_archs(profile):
1044            for bsp in self.config.profile_arch_bsps(profile, arch):
1045                self.build_arch_bsp(arch, bsp)
1046        end = datetime.datetime.now()
1047        log.notice('^ Profile Time %s' % (str(end - start)))
1048        self._finished()
1049
1050def run_args(args):
1051    b = None
1052    ec = 0
1053    try:
1054        #
1055        # On Windows MSYS2 prepends a path to itself to the environment
1056        # path. This means the RTEMS specific automake is not found and which
1057        # breaks the bootstrap. We need to remove the prepended path. Also
1058        # remove any ACLOCAL paths from the environment.
1059        #
1060        if os.name == 'nt':
1061            cspath = os.environ['PATH'].split(os.pathsep)
1062            if 'msys' in cspath[0] and cspath[0].endswith('bin'):
1063                os.environ['PATH'] = os.pathsep.join(cspath[1:])
1064
1065        top = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
1066        prefix = '/opt/rtems/%s' % (rtems_version())
1067        tools = prefix
1068        build_dir = 'bsp-builds'
1069        logf = 'bsp-build-%s.txt' % \
1070               (datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))
1071        config_file = path.join(top, 'share', 'rtems', 'tester',
1072                                'rtems', 'rtems-bsps.ini')
1073        if not path.exists(config_file):
1074            config_file = path.join(top, 'tester', 'rtems', 'rtems-bsps.ini')
1075
1076        argsp = argparse.ArgumentParser()
1077        argsp.add_argument('--prefix', help = 'Prefix to build the BSP.',
1078                           type = str)
1079        argsp.add_argument('--rtems-tools', help = 'The RTEMS tools directory.',
1080                           type = str)
1081        argsp.add_argument('--rtems', help = 'The RTEMS source tree.',
1082                           type = str)
1083        argsp.add_argument('--config-report', help = 'Report the configuration.',
1084                           action = 'store_true')
1085        argsp.add_argument('--warnings-report', help = 'Report the warnings to a file.',
1086                           type = str, default = None)
1087        argsp.add_argument('--build-path', help = 'Path to build in.',
1088                           type = str)
1089        argsp.add_argument('--log', help = 'Log file.', type = str)
1090        argsp.add_argument('--stop-on-error', help = 'Stop on an error.',
1091                           action = 'store_true')
1092        argsp.add_argument('--no-clean', help = 'Do not clean the build output.',
1093                           action = 'store_true')
1094        argsp.add_argument('--profiles', help = 'Build the listed profiles.',
1095                           type = str, default = 'tier-1')
1096        argsp.add_argument('--build', help = 'Build name to build.',
1097                           type = str, default='all')
1098        argsp.add_argument('--arch', help = 'Build the specific architecture.',
1099                           type = str)
1100        argsp.add_argument('--bsp', help = 'Build the specific BSP.',
1101                           type = str)
1102        argsp.add_argument('--jobs', help = 'Number of jobs to run.',
1103                           type = int, default = host.cpus())
1104        argsp.add_argument('--dry-run', help = 'Do not run the actual builds.',
1105                           action = 'store_true')
1106
1107        opts = argsp.parse_args(args[1:])
1108        if opts.log is not None:
1109            logf = opts.log
1110        log.default = log.log([logf])
1111        log.notice(title())
1112        log.output(command_line())
1113
1114        config = configuration()
1115        config.load(config_file, opts.build)
1116
1117        if opts.config_report:
1118            log.notice('Configuration Report:')
1119            log.notice(config.report())
1120            sys.exit(0)
1121
1122        if opts.rtems is None:
1123            raise error.general('No RTEMS source provided on the command line')
1124        if opts.prefix is not None:
1125            prefix = path.shell(opts.prefix)
1126        if opts.rtems_tools is not None:
1127            tools = path.shell(opts.rtems_tools)
1128        if opts.build_path is not None:
1129            build_dir = path.shell(opts.build_path)
1130        if opts.bsp is not None and opts.arch is None:
1131            raise error.general('BSP provided but no architecture')
1132
1133        options = { 'stop-on-error'   : opts.stop_on_error,
1134                    'no-clean'        : opts.no_clean,
1135                    'dry-run'         : opts.dry_run,
1136                    'jobs'            : opts.jobs,
1137                    'warnings-report' : opts.warnings_report }
1138
1139        b = build(config, rtems_version(), prefix, tools,
1140                  path.shell(opts.rtems), build_dir, options)
1141        if opts.arch is not None:
1142            if opts.bsp is not None:
1143                b.build_bsp(opts.arch, opts.bsp)
1144            else:
1145                b.build_arch(opts.arch)
1146        else:
1147            for profile in opts.profiles.split(','):
1148                b.build_profile(profile.strip())
1149
1150    except error.general as gerr:
1151        print(gerr)
1152        print('BSP Build FAILED', file = sys.stderr)
1153        ec = 1
1154    except error.internal as ierr:
1155        print(ierr)
1156        print('BSP Build FAILED', file = sys.stderr)
1157        ec = 1
1158    except error.exit as eerr:
1159        pass
1160    except KeyboardInterrupt:
1161        log.notice('abort: user terminated')
1162        ec = 1
1163    if b is not None:
1164        b.results.report()
1165    sys.exit(ec)
1166
1167def run():
1168    run_args(sys.argv)
1169
1170if __name__ == "__main__":
1171    run()
Note: See TracBrowser for help on using the repository browser.