source: rtems-tools/tester/rt/check.py @ 57e4172

4.105
Last change on this file since 57e4172 was 57e4172, checked in by Chris Johns <chrisj@…>, on 08/18/16 at 04:24:45

tester: handle compile argument warnings.
  • Property mode set to 100755
File size: 19.4 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2016 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# 1. Redistributions of source code must retain the above copyright notice,
12# this list of conditions and the following disclaimer.
13#
14# 2. Redistributions in binary form must reproduce the above copyright notice,
15# this list of conditions and the following disclaimer in the documentation
16# and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31from __future__ import print_function
32
33import argparse
34import datetime
35import operator
36import os
37import sys
38
39from rtemstoolkit import execute
40from rtemstoolkit import error
41from rtemstoolkit import log
42from rtemstoolkit import path
43from rtemstoolkit import version
44
45def rtems_version():
46    return version.version()
47
48class warnings_counter:
49
50    def __init__(self, rtems):
51        self.rtems = path.host(rtems)
52        self.reset()
53
54    def report(self):
55        str = ''
56        sw = sorted(self.warnings.items(), key = operator.itemgetter(1), reverse = True)
57        for w in sw:
58            str += ' %5d %s%s' % (w[1], w[0], os.linesep)
59        return str
60
61    def accumulate(self, total):
62        for w in self.warnings:
63            if w not in total.warnings:
64                total.warnings[w] = self.warnings[w]
65            else:
66                total.warnings[w] += self.warnings[w]
67        total.count += self.count
68
69    def get(self):
70        return self.count
71
72    def reset(self):
73        self.warnings = { }
74        self.count = 0
75
76    def output(self, text):
77        for l in text.splitlines():
78            if ' warning:' in l:
79                self.count += 1
80                ws = l.split(' ')
81                if len(ws) > 0:
82                    ws = ws[0].split(':')
83                    w = path.abspath(ws[0])
84                    w = w.replace(self.rtems, '')
85                    if path.isabspath(w):
86                        w = w[1:]
87                    #
88                    # Ignore compiler option warnings.
89                    #
90                    if len(ws) >= 3:
91                        w = '%s:%s:%s' % (w, ws[1], ws[2])
92                        if w not in self.warnings:
93                            self.warnings[w] = 0
94                        self.warnings[w] += 1
95        log.output(text)
96
97class configuration:
98
99    def __init__(self):
100        try:
101            import configparser
102        except:
103            import ConfigParser as configparser
104        self.config = configparser.ConfigParser()
105        self.name = None
106        self.archs = { }
107        self.builds = { }
108        self.profiles = { }
109
110    def __str__(self):
111        import pprint
112        s = self.name + os.linesep
113        s += 'Archs:' + os.linesep + \
114             pprint.pformat(self.archs, indent = 1, width = 80) + os.linesep
115        s += 'Builds:' + os.linesep + \
116             pprint.pformat(self.builds, indent = 1, width = 80) + os.linesep
117        s += 'Profiles:' + os.linesep + \
118             pprint.pformat(self.profiles, indent = 1, width = 80) + os.linesep
119        return s
120
121    def _get_item(self, section, label, err = True):
122        try:
123            rec = self.config.get(section, label).replace(os.linesep, ' ')
124            return rec
125        except:
126            if err:
127                raise error.general('config: no %s found in %s' % (label, section))
128        return None
129
130    def _comma_list(self, section, label, error = True):
131        items = self._get_item(section, label, error)
132        if items is None:
133            return []
134        return sorted(set([a.strip() for a in items.split(',')]))
135
136    def load(self, name):
137        if not path.exists(name):
138            raise error.general('config: cannot read configuration: %s' % (name))
139        self.name = name
140        self.config.read(name)
141        archs = []
142        self.profiles['profiles'] = self._comma_list('profiles', 'profiles', error = False)
143        if len(self.profiles['profiles']) == 0:
144            self.profiles['profiles'] = ['tier_%d' % (t) for t in range(1,4)]
145        for p in self.profiles['profiles']:
146            profile = {}
147            profile['name'] = p
148            profile['archs'] = self._comma_list(profile['name'], 'archs')
149            archs += profile['archs']
150            for arch in profile['archs']:
151                bsps = 'bsps_%s' % (arch)
152                profile[bsps] = self._comma_list(profile['name'], bsps)
153            self.profiles[profile['name']] = profile
154        for a in set(archs):
155            arch = {}
156            arch['excludes'] = self._comma_list(a, 'excludes', error = False)
157            arch['bsps'] = self._comma_list(a, 'bsps', error = False)
158            for b in arch['bsps']:
159                arch[b] = {}
160                arch[b]['bspopts'] = self._comma_list(a, 'bspopts_%s' % (b), error = False)
161                arch[b]['config'] = self._comma_list(a, 'config_%s' % (b), error = False)
162            self.archs[a] = arch
163        builds = {}
164        builds['default'] = self._get_item('builds', 'default').split()
165        builds['variations'] = self._comma_list('builds', 'variations')
166        builds['var_options'] = {}
167        for v in builds['variations']:
168            builds['var_options'][v] = self._get_item('builds', v).split()
169        self.builds = builds
170
171    def variations(self):
172        return self.builds['variations']
173
174    def excludes(self, arch):
175        return self.archs[arch]['excludes']
176
177    def archs(self):
178        return sorted(self.archs.keys())
179
180    def arch_present(self, arch):
181        return arch in self.archs
182
183    def arch_bsps(self, arch):
184        return sorted(self.archs[arch]['bsps'])
185
186    def bsp_present(self, arch, bsp):
187        return bsp in self.archs[arch]['bsps']
188
189    def bspopts(self, arch, bsp):
190        return self.archs[arch][bsp]['bspopts']
191
192    def defaults(self):
193        return self.builds['default']
194
195    def variant_options(self, variant):
196        if variant in self.builds['var_options']:
197            self.builds['var_options'][variant]
198        return []
199
200    def profile_present(self, profile):
201        return profile in self.profiles
202
203    def profile_archs(self, profile):
204        return self.profiles[profile]['archs']
205
206    def profile_arch_bsps(self, profile, arch):
207        return self.profiles[profile]['bsps_%s' % (arch)]
208
209class build:
210
211    def __init__(self, config, version, prefix, tools, rtems, build_dir, options):
212        self.config = config
213        self.build_dir = build_dir
214        self.rtems_version = version
215        self.prefix = prefix
216        self.tools = tools
217        self.rtems = rtems
218        self.options = options
219        self.errors = { 'configure': 0,
220                        'build':     0,
221                        'tests':     0 }
222        self.counts = { 'h'        : 0,
223                        'exes'     : 0,
224                        'objs'     : 0,
225                        'libs'     : 0 }
226        self.warnings = warnings_counter(rtems)
227        if not path.exists(path.join(rtems, 'configure')) or \
228           not path.exists(path.join(rtems, 'Makefile.in')) or \
229           not path.exists(path.join(rtems, 'cpukit')):
230            raise error.general('RTEMS source path does not look like RTEMS')
231
232    def _error_str(self):
233        return 'Status: configure:%d build:%d' % \
234            (self.errors['configure'], self.errors['build'])
235
236    def _path(self, arch, bsp):
237        return path.join(self.build_dir, arch, bsp)
238
239    def _archs(self, build_data):
240        return sorted(build_data.keys())
241
242    def _bsps(self, arch):
243        return self.config.arch_bsps(arch)
244
245    def _variations(self, arch, bsp):
246        vars = self.config.variations()
247        for v in self.config.excludes(arch):
248            if v in vars:
249                vars.remove(v)
250        return vars
251
252    def _arch_bsp_dir_make(self, arch, bsp):
253        if not path.exists(self._path(arch, bsp)):
254            path.mkdir(self._path(arch, bsp))
255
256    def _arch_bsp_dir_clean(self, arch, bsp):
257        if path.exists(self._path(arch, bsp)):
258            path.removeall(self._path(arch, bsp))
259
260    def _config_command(self, commands, arch, bsp):
261        cmd = [path.join(self.rtems, 'configure')]
262        commands += self.config.bspopts(arch, bsp)
263        for c in commands:
264            c = c.replace('@PREFIX@', self.prefix)
265            c = c.replace('@RTEMS_VERSION@', self.rtems_version)
266            c = c.replace('@ARCH@', arch)
267            c = c.replace('@BSP@', bsp)
268            cmd += [c]
269        return ' '.join(cmd)
270
271    def _build_set(self, variations):
272        build_set = { }
273        bs = self.config.defaults()
274        for var in variations:
275            build_set[var] = bs + self.config.variant_options(var)
276        return build_set
277
278    def _build_dir(self, arch, bsp, build):
279        return path.join(self._path(arch, bsp), build)
280
281    def _count_files(self, arch, bsp, build):
282        counts = { 'h'    : 0,
283                   'exes' : 0,
284                   'objs' : 0,
285                   'libs' : 0 }
286        for root, dirs, files in os.walk(self._build_dir(arch, bsp, build)):
287            for file in files:
288                if file.endswith('.exe'):
289                    counts['exes'] += 1
290                elif file.endswith('.o'):
291                    counts['objs'] += 1
292                elif file.endswith('.a'):
293                    counts['libs'] += 1
294                elif file.endswith('.h'):
295                    counts['h'] += 1
296        for f in self.counts:
297            if f in counts:
298                self.counts[f] = counts[f]
299        return counts
300
301    def build_arch_bsp(self, arch, bsp):
302        if not self.config.bsp_present(arch, bsp):
303            raise error.general('BSP not found: %s/%s' % (arch, bsp))
304        log.output('-' * 70)
305        log.notice('] BSP: %s/%s' % (arch, bsp))
306        log.notice('. Creating: %s' % (self._path(arch, bsp)))
307        self._arch_bsp_dir_clean(arch, bsp)
308        self._arch_bsp_dir_make(arch, bsp)
309        variations = self._variations(arch, bsp)
310        build_set = self._build_set(variations)
311        bsp_start = datetime.datetime.now()
312        bsp_warnings = warnings_counter(self.rtems)
313        env_path = os.environ['PATH']
314        os.environ['PATH'] = path.host(path.join(self.tools, 'bin')) + \
315                             os.pathsep + os.environ['PATH']
316        for bs in sorted(build_set.keys()):
317            warnings = warnings_counter(self.rtems)
318            start = datetime.datetime.now()
319            log.output('- ' * 35)
320            log.notice('. Configuring: %s' % (bs))
321            try:
322                bpath = self._build_dir(arch, bsp, bs)
323                path.mkdir(bpath)
324                cmd = self._config_command(build_set[bs], arch, bsp)
325                e = execute.capture_execution(log = warnings)
326                log.output('run: ' + cmd)
327                if self.options['dry-run']:
328                    exit_code = 0
329                else:
330                    exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath))
331                if exit_code != 0:
332                    self.errors['configure'] += 1
333                    log.notice('- Configure failed: %s' % (bs))
334                    log.output('cmd failed: %s' % (cmd))
335                    if self.options['stop-on-error']:
336                        raise error.general('Configuring %s failed' % (bs))
337                else:
338                    log.notice('. Building: %s' % (bs))
339                    cmd = 'make'
340                    if 'jobs' in self.options:
341                        cmd += ' -j %s' % (self.options['jobs'])
342                    log.output('run: ' + cmd)
343                    if self.options['dry-run']:
344                        exit_code = 0
345                    else:
346                        exit_code, proc, output = e.shell(cmd, cwd = path.host(bpath))
347                    if exit_code != 0:
348                        self.errors['build'] += 1
349                        log.notice('- FAIL: %s: %s' % (bs, self._error_str()))
350                        log.output('cmd failed: %s' % (cmd))
351                        if self.options['stop-on-error']:
352                            raise error.general('Building %s failed' % (bs))
353                    files = self._count_files(arch, bsp, bs)
354                    log.notice('+ Pass: %s: warnings:%d  exes:%d  objs:%s  libs:%d' % \
355                               (bs, warnings.get(),
356                                files['exes'], files['objs'], files['libs']))
357                    log.notice('  %s' % (self._error_str()))
358            finally:
359                end = datetime.datetime.now()
360                if not self.options['no-clean']:
361                    log.notice('. Cleaning: %s' % (self._build_dir(arch, bsp, bs)))
362                    path.removeall(self._build_dir(arch, bsp, bs))
363            log.notice('^ Time %s' % (str(end - start)))
364            log.output('Warnings Report:')
365            log.output(warnings.report())
366            warnings.accumulate(bsp_warnings)
367            warnings.accumulate(self.warnings)
368        bsp_end = datetime.datetime.now()
369        log.notice('^ BSP Time %s' % (str(bsp_end - bsp_start)))
370        log.output('BSP Warnings Report:')
371        log.output(bsp_warnings.report())
372        os.environ['PATH'] = env_path
373
374    def build_arch(self, arch):
375        start = datetime.datetime.now()
376        log.output('=' * 70)
377        log.notice(']] Architecture: %s' % (arch))
378        if not self.confif.arch_present(arch):
379            raise error.general('Architecture not found: %s' % (arch))
380        for bsp in self._bsps(arch):
381            self.build_arch_bsp(arch, bsp)
382        log.notice('^ Architecture Time %s' % (str(end - start)))
383        log.notice('  warnings:%d  exes:%d  objs:%s  libs:%d' % \
384                   self.warnings.get(), self.counts['exes'],
385                   self.counts['objs'], self.counts['libs'])
386        log.output('Architecture Warnings:')
387        log.output(self.warnings.report())
388
389    def build(self):
390        for arch in self.config.archs():
391            self.build_arch(arch)
392        log.notice('^ Profile Time %s' % (str(end - start)))
393        log.notice('+  warnings:%d  exes:%d  objs:%s  libs:%d' % \
394                   self.warnings.get(), self.counts['exes'],
395                   self.counts['objs'], self.counts['libs'])
396        log.output('Profile Warnings:')
397        log.output(self.warnings.report())
398
399    def build_profile(self, profile):
400        if not self.config.profile_present(profile):
401            raise error.general('BSP not found: %s/%s' % (arch, bsp))
402        start = datetime.datetime.now()
403        log.notice(']] Profile: %s' % (profile))
404        for arch in self.config.profile_archs(profile):
405            for bsp in self.config.profile_arch_bsps(profile, arch):
406                self.build_arch_bsp(arch, bsp)
407        end = datetime.datetime.now()
408        log.notice('^ Profile Time %s' % (str(end - start)))
409        log.notice('  warnings:%d  exes:%d  objs:%d  libs:%d' % \
410                   (self.warnings.get(), self.counts['exes'],
411                    self.counts['objs'], self.counts['libs']))
412        log.output('Profile Warnings:')
413        log.output(self.warnings.report())
414
415def run_args(args):
416    try:
417        #
418        # On Windows MSYS2 prepends a path to itself to the environment
419        # path. This means the RTEMS specific automake is not found and which
420        # breaks the bootstrap. We need to remove the prepended path. Also
421        # remove any ACLOCAL paths from the environment.
422        #
423        if os.name == 'nt':
424            cspath = os.environ['PATH'].split(os.pathsep)
425            if 'msys' in cspath[0] and cspath[0].endswith('bin'):
426                os.environ['PATH'] = os.pathsep.join(cspath[1:])
427
428        top = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
429        profile = 'tier-1'
430        prefix = '/opt/rtems/%s' % (rtems_version())
431        tools = prefix
432        build_dir = 'bsp-builds'
433        logf = 'bsp-build-%s.txt' % (datetime.datetime.now().strftime('%Y%m%d-%H%M%S'))
434        config_file = path.join(top, 'share', 'rtems', 'tester', 'rtems', 'rtems-bsps.ini')
435
436        argsp = argparse.ArgumentParser()
437        argsp.add_argument('--prefix', help = 'Prefix to build the BSP.', type = str)
438        argsp.add_argument('--rtems-tools', help = 'The RTEMS tools directory.', type = str)
439        argsp.add_argument('--rtems', help = 'The RTEMS source tree.', type = str)
440        argsp.add_argument('--build-path', help = 'Path to build in.', type = str)
441        argsp.add_argument('--log', help = 'Log file.', type = str)
442        argsp.add_argument('--stop-on-error', help = 'Stop on an error.', action = 'store_true')
443        argsp.add_argument('--no-clean', help = 'Do not clean the build output.', action = 'store_true')
444        argsp.add_argument('--arch', help = 'Build the specific architecture.', type = str)
445        argsp.add_argument('--bsp', help = 'Build the specific BSP.', type = str)
446        argsp.add_argument('--dry-run', help = 'Do not run the actual builds.', action = 'store_true')
447
448        opts = argsp.parse_args(args[1:])
449        if opts.log is not None:
450            logf = opts.log
451        log.default = log.log([logf])
452        log.notice('RTEMS Tools Project - RTEMS Kernel Check, %s' % (version.str()))
453        if opts.rtems is None:
454            raise error.general('No RTEMS source provided on the command line')
455        if opts.prefix is not None:
456            prefix = path.shell(opts.prefix)
457        if opts.rtems_tools is not None:
458            tools = path.shell(opts.rtems_tools)
459        if opts.build_path is not None:
460            build_dir = path.shell(opts.build_path)
461        if opts.bsp is not None and opts.arch is None:
462            raise error.general('BSP provided but no architecture')
463
464        config = configuration()
465        config.load(config_file)
466
467        options = { 'stop-on-error' : opts.stop_on_error,
468                    'no-clean'      : opts.no_clean,
469                    'dry-run'       : opts.dry_run,
470                    'jobs'          : 8 }
471
472        b = build(config, rtems_version(), prefix, tools, path.shell(opts.rtems), build_dir, options)
473        if opts.arch is not None:
474            if opts.bsp is not None:
475                b.build_arch_bsp(opts.arch, opts.bsp)
476            else:
477                b.build_arch(opts.arch)
478        else:
479            b.build_profile(profile)
480
481    except error.general as gerr:
482        print(gerr)
483        print('BSP Build FAILED', file = sys.stderr)
484        sys.exit(1)
485    except error.internal as ierr:
486        print(ierr)
487        print('BSP Build FAILED', file = sys.stderr)
488        sys.exit(1)
489    except error.exit as eerr:
490        pass
491    except KeyboardInterrupt:
492        log.notice('abort: user terminated')
493        sys.exit(1)
494    sys.exit(0)
495
496def run():
497    run_args(sys.argv)
498
499if __name__ == "__main__":
500    run()
Note: See TracBrowser for help on using the repository browser.