source: rtems-source-builder/source-builder/sb/execute.py @ f88fcf3

4.11
Last change on this file since f88fcf3 was f88fcf3, checked in by Chris Johns <chrisj@…>, on 03/07/16 at 00:56:02

sb: Update code base to support Python3 and Python2.

Fix Windows support to allow MSYS2 Python to be used.

Updates #2619.
  • Property mode set to 100755
File size: 14.8 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2016 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# Execute commands or scripts.
22#
23# Note, the subprocess module is only in Python 2.4 or higher.
24#
25
26from __future__ import print_function
27
28import functools
29import os
30import re
31import sys
32import subprocess
33import threading
34
35import error
36import log
37
38# Redefine the PIPE from subprocess
39PIPE = subprocess.PIPE
40
41# Regular expression to find quotes.
42qstr = re.compile('[rR]?\'([^\\n\'\\\\]|\\\\.)*\'|[rR]?"([^\\n"\\\\]|\\\\.)*"')
43
44def check_type(command):
45    """Checks the type of command we have. The types are spawn and
46    shell."""
47    if command in ['spawn', 'shell']:
48        return True
49    return False
50
51def arg_list(args):
52    """Turn a string of arguments into a list suitable for
53    spawning a command. If the args are already a list return
54    it."""
55    if type(args) is list:
56        return args
57    argstr = args
58    args = []
59    while len(argstr):
60        qs = qstr.search(argstr)
61        if not qs:
62            args.extend(argstr.split())
63            argstr= ''
64        else:
65            # We have a quoted string. Get the string before
66            # the quoted string and splt on white space then
67            # add the quoted string as an option then remove
68            # the first + quoted string and try again
69            front = argstr[:qs.start()]
70            args.extend(front.split())
71            args.append(argstr[qs.start() + 1:qs.end() - 1])
72            argstr = argstr[qs.end():]
73    return args
74
75def arg_subst(command, substs):
76    """Substitute the %[0-9] in the command with the subst values."""
77    args = arg_list(command)
78    if substs:
79        for a in range(0, len(args)):
80            for r in range(0, len(substs)):
81                args[a] = re.compile(('%%%d' % (r))).sub(substs[r], args[a])
82    return args
83
84def arg_subst_str(command, subst):
85    cmd = arg_subst(command, subst)
86    def add(x, y): return x + ' ' + str(y)
87    return functools.reduce(add, cmd, '')
88
89class execute:
90    """Execute commands or scripts. The 'output' is a funtion
91    that handles the output from the process."""
92    def __init__(self, output = None, error_prefix = '', verbose = False):
93        self.output = output
94        self.error_prefix = error_prefix
95        self.verbose = verbose
96        self.shell_exe = None
97        self.shell_commands = False
98        self.path = None
99        self.environment = None
100
101    def capture(self, proc, timeout = None):
102        """Create 2 threads to read stdout and stderr and send to the
103        output handler. Based on the 'communicate' code in the subprocess
104        module."""
105        def _readthread(fh, out, prefix = ''):
106            """Read from a file handle and write to the output handler
107            until the file closes."""
108            count = 0
109            while True:
110                line = fh.readline()
111                # str and bytes are the same type in Python2
112                if type(line) is not str and type(line) is bytes:
113                    line = line.decode(sys.stdout.encoding)
114                count += 1
115                if len(line) == 0:
116                    break
117                if out:
118                    out(prefix + line)
119                else:
120                    log.output(prefix + line)
121                    if count > 10:
122                        log.flush()
123                        count = 0
124
125        def _timerthread(proc, timer):
126            """Timer thread calls the timer handler if one
127            is present once a second. The user provides a handler
128            and returns False to kill the process or True continue."""
129            while True:
130                time.sleep(1)
131                if not timer(proc):
132                    proc.stdout.close()
133                    proc.stderr.close()
134
135        if proc.stdout:
136            stdout_thread = threading.Thread(target = _readthread,
137                                             args = (proc.stdout,
138                                                     self.output,
139                                                     ''))
140            stdout_thread.setDaemon(True)
141            stdout_thread.start()
142        if proc.stderr:
143            stderr_thread = threading.Thread(target = _readthread,
144                                             args = (proc.stderr,
145                                                     self.output,
146                                                     self.error_prefix))
147            stderr_thread.setDaemon(True)
148            stderr_thread.start()
149        if proc.stdout:
150            stdout_thread.join()
151        if proc.stderr:
152            stderr_thread.join()
153        return proc.wait()
154
155    def open(self, command, capture = True, shell = False,
156             cwd = None, env = None,
157             stdin = None, stdout = None, stderr = None):
158        """Open a command with arguments. Provide the arguments as a list or
159        a string."""
160        if self.verbose:
161            s = command
162            if type(command) is list:
163                def add(x, y): return x + ' ' + str(y)
164                s = functools.reduce(add, command, '')[1:]
165            what = 'spawn'
166            if shell:
167                what = 'shell'
168            log.output(what + ': ' + s)
169        if shell and self.shell_exe:
170            command = arg_list(command)
171            command[:0] = self.shell_exe
172        if not stdout:
173            stdout = subprocess.PIPE
174        if not stderr:
175            stderr = subprocess.PIPE
176        proc = None
177        if cwd is None:
178            cwd = self.path
179        if env is None:
180            env = self.environment
181        try:
182            # Work around a problem on Windows with commands that
183            # have a '.' and no extension. Windows needs the full
184            # command name.
185            if sys.platform == "win32" and type(command) is list:
186                if command[0].find('.') >= 0:
187                    r, e = os.path.splitext(command[0])
188                    if e not in ['.exe', '.com', '.bat']:
189                        command[0] = command[0] + '.exe'
190            log.trace('exe: %s' % (command))
191            proc = subprocess.Popen(command, shell = shell,
192                                    cwd = cwd, env = env,
193                                    stdin = stdin, stdout = stdout,
194                                    stderr = stderr)
195            if not capture:
196                return (0, proc)
197            exit_code = self.capture(proc)
198            if self.verbose:
199                log.output('exit: ' + str(exit_code))
200        except OSError as ose:
201            exit_code = ose.errno
202            if self.verbose:
203                log.output('exit: ' + str(ose))
204        return (exit_code, proc)
205
206    def spawn(self, command, capture = True, cwd = None, env = None,
207              stdin = None, stdout = None, stderr = None):
208        """Spawn a command with arguments. Provide the arguments as a list or
209        a string."""
210        return self.open(command, capture, False, cwd, env,
211                         stdin, stdout, stderr)
212
213    def shell(self, command, capture = True, cwd = None, env = None,
214              stdin = None, stdout = None, stderr = None):
215        """Execute a command within a shell context. The command can contain
216        argumments. The shell is specific to the operating system. For example
217        it is cmd.exe on Windows XP."""
218        return self.open(command, capture, True, cwd, env,
219                         stdin, stdout, stderr)
220
221    def command(self, command, args = None, capture = True, shell = False,
222                cwd = None, env = None,
223                stdin = None, stdout = None, stderr = None):
224        """Run the command with the args. The args can be a list
225        or a string."""
226        if args and not type(args) is list:
227            args = arg_list(args)
228        cmd = [command]
229        if args:
230            cmd.extend(args)
231        return self.open(cmd, capture = capture, shell = shell,
232                         cwd = cwd, env = env,
233                         stdin = stdin, stdout = stdout, stderr = stderr)
234
235    def command_subst(self, command, substs, capture = True, shell = False,
236                      cwd = None, env = None,
237                      stdin = None, stdout = None, stderr = None):
238        """Run the command from the config data with the
239        option format string subsituted with the subst variables."""
240        args = arg_subst(command, substs)
241        return self.command(args[0], args[1:], capture = capture,
242                            shell = shell or self.shell_commands,
243                            cwd = cwd, env = env,
244                            stdin = stdin, stdout = stdout, stderr = stderr)
245
246    def set_shell(self, execute):
247        """Set the shell to execute when issuing a shell command."""
248        args = arg_list(execute)
249        if len(args) == 0 or not os.path.isfile(args[0]):
250            raise error.general('could find shell: ' + execute)
251        self.shell_exe = args
252
253    def command_use_shell(self):
254        """Force all commands to use a shell. This can be used with set_shell
255        to allow Unix commands be executed on Windows with a Unix shell such
256        as Cygwin or MSYS. This may cause piping to fail."""
257        self.shell_commands = True
258
259    def set_output(self, output):
260        """Set the output handler. The stdout of the last process in a pipe
261        line is passed to this handler."""
262        old_output = self.output
263        self.output = output
264        return old_output
265
266    def set_path(self, path):
267        """Set the path changed to before the child process is created."""
268        old_path = self.path
269        self.path = path
270        return old_path
271
272    def set_environ(self, environment):
273        """Set the environment passed to the child process when created."""
274        old_environment = self.environment
275        self.environment = environment
276        return old_environment
277
278class capture_execution(execute):
279    """Capture all output as a string and return it."""
280
281    class _output_snapper:
282        def __init__(self, log = None, dump = False):
283            self.output = ''
284            self.log = log
285            self.dump = dump
286
287        def handler(self, text):
288            if not self.dump:
289                if self.log is not None:
290                    self.log.output(text)
291                else:
292                    self.output += text
293
294        def get_and_clear(self):
295            text = self.output
296            self.output = ''
297            return text.strip()
298
299    def __init__(self, log = None, dump = False, error_prefix = '', verbose = False):
300        self.snapper = capture_execution._output_snapper(log = log, dump = dump)
301        execute.__init__(self, output = self.snapper.handler,
302                         error_prefix = error_prefix,
303                         verbose = verbose)
304
305    def open(self, command, capture = True, shell = False, cwd = None, env = None,
306             stdin = None, stdout = None, stderr = None):
307        if not capture:
308            raise error.general('output capture must be true; leave as default')
309        #self.snapper.get_and_clear()
310        exit_code, proc = execute.open(self, command, capture = True, shell = shell,
311                                       cwd = cwd, env = env,
312                                       stdin = stdin, stdout = stdout, stderr = stderr)
313        return (exit_code, proc, self.snapper.get_and_clear())
314
315    def set_output(self, output):
316        raise error.general('output capture cannot be overrided')
317
318if __name__ == "__main__":
319    def run_tests(e, commands, use_shell):
320        for c in commands['shell']:
321            e.shell(c)
322        for c in commands['spawn']:
323            e.spawn(c)
324        for c in commands['cmd']:
325            if type(c) is str:
326                e.command(c, shell = use_shell)
327            else:
328                e.command(c[0], c[1], shell = use_shell)
329        for c in commands['csubsts']:
330            e.command_subst(c[0], c[1], shell = use_shell)
331        ec, proc = e.command(commands['pipe'][0], commands['pipe'][1],
332                             capture = False, stdin = subprocess.PIPE)
333        if ec == 0:
334            print('piping input into ' + commands['pipe'][0] + ': ' + \
335                  commands['pipe'][2])
336            proc.stdin.write(bytes(commands['pipe'][2], sys.stdin.encoding))
337            proc.stdin.close()
338            e.capture(proc)
339            del proc
340
341    cmd_shell_test = 'if "%OS%" == "Windows_NT" (echo It is WinNT) else echo Is is not WinNT'
342    sh_shell_test = 'x="me"; if [ $x = "me" ]; then echo "It was me"; else "It was him"; fi'
343
344    commands = {}
345    commands['windows'] = {}
346    commands['unix'] = {}
347    commands['windows']['shell'] = ['cd', 'dir /w', '.\\xyz', cmd_shell_test]
348    commands['windows']['spawn'] = ['hostname', 'hostnameZZ', ['netstat', '/e']]
349    commands['windows']['cmd'] = [('ipconfig'), ('nslookup', 'www.python.org')]
350    commands['windows']['csubsts'] = [('netstat %0', ['-a']),
351                                      ('netstat %0 %1', ['-a', '-n'])]
352    commands['windows']['pipe'] = ('ftp', None, 'help\nquit')
353    commands['unix']['shell'] = ['pwd', 'ls -las', './xyz', sh_shell_test]
354    commands['unix']['spawn'] = ['ls', 'execute.pyc', ['ls', '-i']]
355    commands['unix']['cmd'] = [('date'), ('date', '-R'), ('date', ['-u', '+%d %D']),
356                               ('date', '-u "+%d %D %S"')]
357    commands['unix']['csubsts'] = [('date %0 "+%d %D %S"', ['-u']),
358                                   ('date %0 %1', ['-u', '+%d %D %S'])]
359    commands['unix']['pipe'] = ('grep', 'hello', 'hello world')
360
361    print(arg_list('cmd a1 a2 "a3 is a string" a4'))
362    print(arg_list('cmd b1 b2 "b3 is a string a4'))
363    print(arg_subst(['nothing', 'xx-%0-yyy', '%1', '%2-something'],
364                    ['subst0', 'subst1', 'subst2']))
365
366    e = execute(error_prefix = 'ERR: ', verbose = True)
367    if sys.platform == "win32":
368        run_tests(e, commands['windows'], False)
369        if os.path.exists('c:\\msys\\1.0\\bin\\sh.exe'):
370            e.set_shell('c:\\msys\\1.0\\bin\\sh.exe --login -c')
371            commands['unix']['pipe'] = ('c:\\msys\\1.0\\bin\\grep',
372                                        'hello', 'hello world')
373            run_tests(e, commands['unix'], True)
374    else:
375        run_tests(e, commands['unix'], False)
376    del e
Note: See TracBrowser for help on using the repository browser.