source: rtems-tools/tester/rt/report.py @ a4fbd16

5
Last change on this file since a4fbd16 was a4fbd16, checked in by Chris Johns <chrisj@…>, on 11/26/18 at 03:38:37

tester: Fix the handling of the version and tools recs when running parallel jobs

  • Property mode set to 100644
File size: 14.1 KB
RevLine 
[50fdf12]1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2013-2014 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# 1. Redistributions of source code must retain the above copyright notice,
12# this list of conditions and the following disclaimer.
13#
14# 2. Redistributions in binary form must reproduce the above copyright notice,
15# this list of conditions and the following disclaimer in the documentation
16# and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31#
32# RTEMS Testing Reports
33#
34
[a4fbd16]35from __future__ import print_function
36
[50fdf12]37import datetime
38import os
39import threading
40
41from rtemstoolkit import error
42from rtemstoolkit import log
43from rtemstoolkit import path
44
[3a867a4]45#
46# Maybe this should be a configuration.
47#
48test_fail_excludes = [
49    'minimum'
50]
51
[50fdf12]52class report(object):
53    '''RTEMS Testing report.'''
54
55    def __init__(self, total):
56        self.lock = threading.Lock()
57        self.total = total
58        self.total_len = len(str(total))
59        self.passed = 0
60        self.failed = 0
[30218f5]61        self.user_input = 0
[15a3e06]62        self.expected_fail = 0
63        self.indeterminate = 0
64        self.benchmark = 0
[50fdf12]65        self.timeouts = 0
66        self.invalids = 0
[30218f5]67        self.wrong_version = 0
68        self.wrong_build = 0
69        self.wrong_tools = 0
[50fdf12]70        self.results = {}
[30218f5]71        self.config = {}
[50fdf12]72        self.name_max_len = 0
73
74    def __str__(self):
[15a3e06]75        msg  = 'Passed:        %*d%s' % (self.total_len, self.passed, os.linesep)
76        msg += 'Failed:        %*d%s' % (self.total_len, self.failed, os.linesep)
77        msg += 'User Input:    %*d%s' % (self.total_len, self.user_input, os.linesep)
78        msg += 'Expected Fail: %*d%s' % (self.total_len, self.expected_fail, os.linesep)
79        msg += 'Indeterminate: %*d%s' % (self.total_len, self.self.indeterminate, os.linesep)
80        msg += 'Benchmark:     %*d%s' % (self.total_len, self.self.benchmark, os.linesep)
81        msg += 'Timeout:       %*d%s' % (self.total_len, self.timeouts, os.linesep)
82        msg += 'Invalid:       %*d%s' % (self.total_len, self.invalids, os.linesep)
[30218f5]83        msg += 'Wrong Version  %*d%s' % (self.total_len, self.wrong_version, os.linesep)
84        msg += 'Wrong Build    %*d%s' % (self.total_len, self.wrong_build, os.linesep)
85        msg += 'Wrong Tools    %*d%s' % (self.total_len, self.wrong_tools, os.linesep)
[50fdf12]86        return msg
87
[30218f5]88    def start(self, index, total, name, executable, bsp_arch, bsp, show_header):
89        header = '[%*d/%*d] p:%-*d f:%-*d u:%-*d e:%-*d I:%-*d B:%-*d ' \
90                 't:%-*d i:%-*d W:%-*d | %s/%s: %s' % \
[50fdf12]91                 (len(str(total)), index,
92                  len(str(total)), total,
93                  len(str(total)), self.passed,
94                  len(str(total)), self.failed,
[15a3e06]95                  len(str(total)), self.user_input,
96                  len(str(total)), self.expected_fail,
97                  len(str(total)), self.indeterminate,
98                  len(str(total)), self.benchmark,
[50fdf12]99                  len(str(total)), self.timeouts,
100                  len(str(total)), self.invalids,
[30218f5]101                  len(str(total)), self.wrong_version + self.wrong_build + self.wrong_tools,
[50fdf12]102                  bsp_arch,
103                  bsp,
[5251361]104                  path.basename(name))
[50fdf12]105        self.lock.acquire()
106        if name in self.results:
107            self.lock.release()
108            raise error.general('duplicate test: %s' % (name))
109        self.results[name] = { 'index': index,
110                               'bsp': bsp,
111                               'bsp_arch': bsp_arch,
[bf58911]112                               'exe': name,
[50fdf12]113                               'start': datetime.datetime.now(),
114                               'end': None,
115                               'result': None,
116                               'output': None,
117                               'header': header }
118
119        self.lock.release()
[30218f5]120        if show_header:
121            log.notice(header, stdout_only = True)
[50fdf12]122
[30218f5]123    def end(self, name, output, output_prefix):
[50fdf12]124        start = False
125        end = False
[15a3e06]126        state = None
[30218f5]127        version = None
128        build = None
129        tools = None
[50fdf12]130        timeout = False
131        prefixed_output = []
132        for line in output:
[30218f5]133            if line[0] == output_prefix:
[50fdf12]134                if line[1].startswith('*** '):
[30218f5]135                    banner = line[1][4:]
136                    if banner.startswith('BEGIN OF '):
[3a867a4]137                        start = True
[30218f5]138                    elif line[1][4:].startswith('END OF '):
[50fdf12]139                        end = True
[30218f5]140                    elif banner.startswith('TIMEOUT TIMEOUT'):
[50fdf12]141                        timeout = True
[30218f5]142                    elif banner.startswith('TEST VERSION:'):
143                        version = banner[13:].strip()
144                    elif banner.startswith('TEST STATE:'):
145                        state = banner[11:].strip()
146                    elif banner.startswith('TEST BUILD:'):
147                        build = ','.join(banner[11:].strip().split(' '))
148                    elif banner.startswith('TEST TOOLS:'):
149                        tools = banner[11:].strip()
[50fdf12]150            prefixed_output += [line[0] + ' ' + line[1]]
151        self.lock.acquire()
[a4fbd16]152        try:
153            if name not in self.results:
154                raise error.general('test report missing: %s' % (name))
155            if self.results[name]['end'] is not None:
156                raise error.general('test already finished: %s' % (name))
157            self.results[name]['end'] = datetime.datetime.now()
158            if state is not None and state not in ['BENCHMARK',
159                                                   'EXPECTED_FAIL',
160                                                   'INDETERMINATE',
161                                                   'USER_INPUT']:
162                if version:
163                    if 'version' not in self.config:
164                        self.config['version'] = version
165                    else:
166                        if version != self.config['version']:
167                            state = 'WRONG-VERSION'
168                if build:
169                    if 'build' not in self.config:
170                        self.config['build'] = build
171                    else:
172                        if build != self.config['build']:
173                            state = 'WRONG-BUILD'
174                if tools:
175                    if 'tools' not in self.config:
176                        self.config['tools'] = tools
177                    else:
178                        if tools != self.config['tools']:
179                            state = 'WRONG-TOOLS'
180            if state is None or state == 'EXPECTED-PASS':
[15a3e06]181                if start and end:
[a4fbd16]182                    if state is None or state == 'EXPECTED-PASS':
183                        status = 'passed'
184                        self.passed += 1
185                elif timeout:
186                    status = 'timeout'
187                    self.timeouts += 1
188                elif start:
189                    if not end:
190                        status = 'failed'
191                        self.failed += 1
[15a3e06]192                else:
[a4fbd16]193                    exe_name = path.basename(name).split('.')[0]
194                    if exe_name in test_fail_excludes:
195                        status = 'passed'
196                        self.passed += 1
197                    else:
198                        status = 'invalid'
199                        self.invalids += 1
[15a3e06]200            else:
[a4fbd16]201                if state == 'EXPECTED_FAIL':
202                    if start and end:
203                        status = 'passed'
204                        self.passed += 1
205                    else:
206                        status = 'expected-fail'
207                        self.expected_fail += 1
208                elif state == 'USER_INPUT':
209                    status = 'user-input'
210                    self.user_input += 1
211                elif state == 'INDETERMINATE':
212                    if start and end:
213                        status = 'passed'
214                        self.passed += 1
215                    else:
216                        status = 'indeterminate'
217                        self.indeterminate += 1
218                elif state == 'BENCHMARK':
219                    status = 'benchmark'
220                    self.benchmark += 1
221                elif state == 'WRONG-VERSION':
222                    status = 'wrong-version'
223                    self.wrong_version += 1
224                elif state == 'WRONG-BUILD':
225                    status = 'wrong-build'
226                    self.wrong_build += 1
227                elif state == 'WRONG-TOOLS':
228                    status = 'wrong-tools'
229                    self.wrong_tools += 1
230                else:
231                    raise error.general('invalid test state: %s: %s' % (name, state))
232            self.results[name]['result'] = status
233            self.results[name]['output'] = prefixed_output
234            if self.name_max_len < len(path.basename(name)):
235                self.name_max_len = len(path.basename(name))
236        finally:
237            self.lock.release()
[3a867a4]238        return status
[50fdf12]239
240    def log(self, name, mode):
[30218f5]241        status_fails = ['failed', 'timeout', 'invalid',
242                        'wrong-version', 'wrong-build', 'wrong-tools']
[50fdf12]243        if mode != 'none':
244            self.lock.acquire()
245            if name not in self.results:
246                self.lock.release()
247                raise error.general('test report missing: %s' % (name))
[6f8b07c]248            exe = path.basename(self.results[name]['exe'])
[50fdf12]249            result = self.results[name]['result']
250            time = self.results[name]['end'] - self.results[name]['start']
[30218f5]251            failed = result in status_fails
[0eb8400]252            result = 'Result: %-10s Time: %s %s' % (result, str(time), exe)
[50fdf12]253            if mode != 'none':
254                header = self.results[name]['header']
[0eb8400]255            if mode == 'all' or failed:
[50fdf12]256                output = self.results[name]['output']
257            else:
258                output = None
259            self.lock.release()
260            if header:
261                log.output(header)
262            if output:
[0eb8400]263                log.output(result)
[50fdf12]264                log.output(output)
265
[30218f5]266    def get_config(self, config, not_found = None):
267        if config in self.config:
268            return self.config[config]
269        return not_found
270
[24d51f7]271    def score_card(self, mode = 'full'):
272        if mode == 'short':
[30218f5]273            wrongs = self.wrong_version + self.wrong_build + self.wrong_tools
274            return 'Passed:%d Failed:%d Timeout:%d Invalid:%d Wrong:%d' % \
275                (self.passed, self.failed, self.timeouts, self.invalids, wrongs)
[24d51f7]276        elif mode == 'full':
277            l = []
278            l += ['Passed:        %*d' % (self.total_len, self.passed)]
279            l += ['Failed:        %*d' % (self.total_len, self.failed)]
280            l += ['User Input:    %*d' % (self.total_len, self.user_input)]
281            l += ['Expected Fail: %*d' % (self.total_len, self.expected_fail)]
282            l += ['Indeterminate: %*d' % (self.total_len, self.indeterminate)]
283            l += ['Benchmark:     %*d' % (self.total_len, self.benchmark)]
284            l += ['Timeout:       %*d' % (self.total_len, self.timeouts)]
285            l += ['Invalid:       %*d' % (self.total_len, self.invalids)]
[30218f5]286            l += ['Wrong Version: %*d' % (self.total_len, self.wrong_version)]
287            l += ['Wrong Build:   %*d' % (self.total_len, self.wrong_build)]
288            l += ['Wrong Tools:   %*d' % (self.total_len, self.wrong_tools)]
[24d51f7]289            l += ['---------------%s' % ('-' * self.total_len)]
290            l += ['Total:         %*d' % (self.total_len, self.total)]
291            return os.linesep.join(l)
292        raise error.general('invalid socre card mode: %s' % (mode))
[89e8c2a]293
294    def failures(self):
[50fdf12]295        def show_state(results, state, max_len):
[89e8c2a]296            l = []
[50fdf12]297            for name in results:
298                if results[name]['result'] == state:
[89e8c2a]299                    l += [' %s' % (path.basename(name))]
300            return l
301        l = []
[50fdf12]302        if self.failed:
[89e8c2a]303            l += ['Failures:']
304            l += show_state(self.results, 'failed', self.name_max_len)
[15a3e06]305        if self.user_input:
[89e8c2a]306            l += ['User Input:']
307            l += show_state(self.results, 'user-input', self.name_max_len)
[15a3e06]308        if self.expected_fail:
[89e8c2a]309            l += ['Expected Fail:']
310            l += show_state(self.results, 'expected-fail', self.name_max_len)
[15a3e06]311        if self.indeterminate:
[89e8c2a]312            l += ['Indeterminate:']
313            l += show_state(self.results, 'indeterminate', self.name_max_len)
[15a3e06]314        if self.benchmark:
[89e8c2a]315            l += ['Benchmark:']
316            l += show_state(self.results, 'benchmark', self.name_max_len)
[50fdf12]317        if self.timeouts:
[89e8c2a]318            l += ['Timeouts:']
319            l += show_state(self.results, 'timeout', self.name_max_len)
[50fdf12]320        if self.invalids:
[89e8c2a]321            l += ['Invalid:']
322            l += show_state(self.results, 'invalid', self.name_max_len)
[30218f5]323        if self.wrong_version:
324            l += ['Wrong Version:']
325            l += show_state(self.results, 'wrong-version', self.name_max_len)
326        if self.wrong_build:
327            l += ['Wrong Build:']
328            l += show_state(self.results, 'wrong-build', self.name_max_len)
329        if self.wrong_tools:
330            l += ['Wrong Tools:']
331            l += show_state(self.results, 'wrong-tools', self.name_max_len)
[89e8c2a]332        return os.linesep.join(l)
333
334    def summary(self):
335        log.output()
336        log.notice(self.score_card())
337        log.output(self.failures())
Note: See TracBrowser for help on using the repository browser.