source: rtems/rtems-bootstrap @ 02c9eb85

Last change on this file since 02c9eb85 was 02ed0b8, checked in by Chris Johns <chrisj@…>, on 09/03/19 at 00:40:46

Add a parallel bootstrap command.

  • Property mode set to 100755
File size: 8.5 KB
Line 
1#! /usr/bin/env python
2
3#
4# SPDX-License-Identifier: BSD-2-Clause
5#
6# Copyright (C) 2013-2019 Chris Johns (chrisj@rtems.org)
7# All rights reserved.
8#
9# Redistribution and use in source and binary forms, with or without
10# modification, are permitted provided that the following conditions
11# are met:
12# 1. Redistributions of source code must retain the above copyright
13#    notice, this list of conditions and the following disclaimer.
14# 2. Redistributions in binary form must reproduce the above copyright
15#    notice, this list of conditions and the following disclaimer in the
16#    documentation and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31#
32# RTEMS Tools Project (http://www.rtems.org/)
33#
34
35from __future__ import print_function
36
37import argparse
38import datetime
39import multiprocessing
40import os
41import re
42import sys
43import threading
44import time
45
46version = "1.0"
47
48class error(Exception):
49    """Base class for Builder exceptions."""
50    def set_output(self, msg):
51        self.msg = msg
52    def __str__(self):
53        return self.msg
54
55class general_error(error):
56    """Raise for a general error."""
57    def __init__(self, what):
58        self.set_output('error: ' + str(what))
59
60def _collect(path_, file):
61    confs = []
62    for root, dirs, files in os.walk(path_, topdown = True):
63        for f in files:
64            if f == file:
65                confs += [os.path.join(root, f)]
66    return confs
67
68def _grep(file, pattern):
69    rege = re.compile(pattern)
70    try:
71        f = open(file, 'r')
72        matches = [rege.match(l) != None for l in f.readlines()]
73        f.close()
74    except IOError as err:
75        raise general_error('reading: %s' % (file))
76    return True in matches
77
78class command:
79
80    def __init__(self, cmd, cwd):
81        self.exit_code = 0
82        self.thread = None
83        self.output = None
84        self.cmd = cmd
85        self.cwd = cwd
86        self.result = None
87
88    def runner(self):
89
90        import subprocess
91
92        #
93        # Support Python 2.6
94        #
95        if "check_output" not in dir(subprocess):
96            def f(*popenargs, **kwargs):
97                if 'stdout' in kwargs:
98                    raise ValueError('stdout argument not allowed, it will be overridden.')
99                process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
100                output, unused_err = process.communicate()
101                retcode = process.poll()
102                if retcode:
103                    cmd = kwargs.get("args")
104                    if cmd is None:
105                        cmd = popenargs[0]
106                    raise subprocess.CalledProcessError(retcode, cmd)
107                return output
108            subprocess.check_output = f
109
110        self.start_time = datetime.datetime.now()
111        self.exit_code = 0
112        try:
113            try:
114                if os.name == 'nt':
115                    cmd = ['sh', '-c'] + self.cmd
116                else:
117                    cmd = self.cmd
118                self.output = subprocess.check_output(cmd, cwd = self.cwd)
119            except subprocess.CalledProcessError as cpe:
120                self.exit_code = cpe.returncode
121                self.output = cpe.output
122            except OSError as ose:
123                raise general_error('bootstrap failed: %s in %s: %s' % \
124                                        (' '.join(cmd), self.cwd, (str(ose))))
125            except KeyboardInterrupt:
126                pass
127            except:
128                raise
129        except:
130            self.result = sys.exc_info()
131        self.end_time = datetime.datetime.now()
132
133    def run(self):
134        self.thread = threading.Thread(target = self.runner)
135        self.thread.start()
136
137    def is_alive(self):
138        return self.thread and self.thread.is_alive()
139
140    def reraise(self):
141        if self.result is not None:
142            raise self.result[0](self.result[1])
143
144class autoreconf:
145
146    def __init__(self, topdir, configure):
147        self.topdir = topdir
148        self.configure = configure
149        self.cwd = os.path.dirname(self.configure)
150        self.command = command(['autoreconf', '-i', '--no-recursive'], self.cwd)
151        self.command.run()
152
153    def is_alive(self):
154        return self.command.is_alive()
155
156    def post_process(self):
157        if self.command is not None:
158            self.command.reraise()
159            if self.command.exit_code != 0:
160                raise general_error('error: autoreconf: %s' % (' '.join(self.command.cmd)))
161            makefile = os.path.join(self.cwd, 'Makefile.am')
162            if os.path.exists(makefile):
163                if _grep(makefile, 'stamp-h\.in'):
164                    stamp_h = os.path.join(self.cwd, 'stamp-h.in')
165                    try:
166                        t = open(os.path.host(stamp_h), 'w')
167                        t.write('timestamp')
168                        t.close()
169                    except IOError as err:
170                        raise general_error('writing: %s' % (stamp_h))
171
172def generate(topdir, jobs):
173    if type(jobs) is str:
174        jobs = int(jobs)
175    start_time = datetime.datetime.now()
176    confs = _collect(topdir, 'configure.ac')
177    next = 0
178    autoreconfs = []
179    while next < len(confs) or len(autoreconfs) > 0:
180        if next < len(confs) and len(autoreconfs) < jobs:
181            print('%3d/%3d: autoreconf: %s' % \
182                  (next + 1, len(confs), confs[next][len(topdir) + 1:]))
183            autoreconfs += [autoreconf(topdir, confs[next])]
184            next += 1
185        else:
186            for ac in autoreconfs:
187                if not ac.is_alive():
188                    ac.post_process()
189                    autoreconfs.remove(ac)
190                    del ac
191            if len(autoreconfs) >= jobs:
192                time.sleep(1)
193    end_time = datetime.datetime.now()
194    print('Bootstrap time: %s' % (str(end_time - start_time)))
195
196def run(args):
197    try:
198        #
199        # On Windows MSYS2 prepends a path to itself to the environment
200        # path. This means the RTEMS specific automake is not found and which
201        # breaks the bootstrap. We need to remove the prepended path. Also
202        # remove any ACLOCAL paths from the environment.
203        #
204        if os.name == 'nt':
205            cspath = os.environ['PATH'].split(os.pathsep)
206            if 'msys' in cspath[0] and cspath[0].endswith('bin'):
207                os.environ['PATH'] = os.pathsep.join(cspath[1:])
208            if 'ACLOCAL_PATH' in os.environ:
209                #
210                # The clear fails on a current MSYS2 python (Feb 2016). Delete
211                # the entry if the clear fails.
212                #
213                try:
214                    os.environ['ACLOCAL_PATH'].clear()
215                except:
216                    del os.environ['ACLOCAL_PATH']
217
218        argsp = argparse.ArgumentParser(prog = 'rtems-bootstrap',
219                                        description = "Bootstrap in parallel")
220        argsp.add_argument('-j', '--jobs',
221                           help = 'number of jobs to run (default: %(default)s).',
222                           type = int, default = multiprocessing.cpu_count())
223        argsp.add_argument('-r', '--rtems',
224                           type = str, default = os.getcwd(),
225                           help = 'path to the rtems kernel source (default: %(default)s).')
226        argopts = argsp.parse_args(args[1:])
227
228        print('RTEMS Bootstrap, %s' % (version))
229
230        if not os.path.exists(argopts.rtems):
231            raise general_error('path does not exist: %s' % (argopts.rtems))
232        if not os.path.isdir(argopts.rtems):
233            raise general_error('path not a directory: %s' % (argopts.rtems))
234
235        generate(argopts.rtems, argopts.jobs)
236    except general_error as gerr:
237        print(gerr)
238        print('Bootstrap FAILED', file = sys.stderr)
239        sys.exit(1)
240    except KeyboardInterrupt:
241        log.notice('abort: user terminated')
242        sys.exit(1)
243    sys.exit(0)
244
245if __name__ == "__main__":
246    run(sys.argv)
Note: See TracBrowser for help on using the repository browser.