source: rtems-tools/rtemstoolkit/execute.py @ 7d3350d

5
Last change on this file since 7d3350d was 6f8b07c, checked in by Chris Johns <chrisj@…>, on 03/14/16 at 03:59:11

rtemstoolkit: Fix execute's writer thread to not eval() the input.

The conversion to Python3 added an eval() call which is wrong.

Fix the spelling in execute.

Fix labels in the tester gdb locking.

Check the debug-trace arguments.

Close #2642.
  • Property mode set to 100755
File size: 22.4 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2016 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# 1. Redistributions of source code must retain the above copyright notice,
12# this list of conditions and the following disclaimer.
13#
14# 2. Redistributions in binary form must reproduce the above copyright notice,
15# this list of conditions and the following disclaimer in the documentation
16# and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31#
32# Execute commands or scripts.
33#
34# Note, the subprocess module is only in Python 2.4 or higher.
35#
36
37from __future__ import print_function
38
39import functools
40import os
41import re
42import sys
43import subprocess
44import threading
45import time
46import traceback
47
48#
49# Support to handle use in a package and as a unit test.
50# If there is a better way to let us know.
51#
52try:
53    from . import error
54    from . import log
55except (ValueError, SystemError):
56    import error
57    import log
58
59# Trace exceptions
60trace_threads = False
61
62# Redefine the PIPE from subprocess
63PIPE = subprocess.PIPE
64
65# Regular expression to find quotes.
66qstr = re.compile('[rR]?\'([^\\n\'\\\\]|\\\\.)*\'|[rR]?"([^\\n"\\\\]|\\\\.)*"')
67
68def check_type(command):
69    """Checks the type of command we have. The types are spawn and
70    shell."""
71    if command in ['spawn', 'shell']:
72        return True
73    return False
74
75def arg_list(args):
76    """Turn a string of arguments into a list suitable for
77    spawning a command. If the args are already a list return
78    it."""
79    if type(args) is list:
80        return args
81    argstr = args
82    args = []
83    while len(argstr):
84        qs = qstr.search(argstr)
85        if not qs:
86            args.extend(argstr.split())
87            argstr= ''
88        else:
89            # We have a quoted string. Get the string before
90            # the quoted string and splt on white space then
91            # add the quoted string as an option then remove
92            # the first + quoted string and try again
93            front = argstr[:qs.start()]
94            args.extend(front.split())
95            args.append(argstr[qs.start() + 1:qs.end() - 1])
96            argstr = argstr[qs.end():]
97    return args
98
99def arg_subst(command, substs):
100    """Substitute the %[0-9] in the command with the subst values."""
101    args = arg_list(command)
102    if substs:
103        for a in range(0, len(args)):
104            for r in range(0, len(substs)):
105                args[a] = re.compile(('%%%d' % (r))).sub(substs[r], args[a])
106    return args
107
108def arg_subst_str(command, subst):
109    cmd = arg_subst(command, subst)
110    def add(x, y): return x + ' ' + str(y)
111    return functools.reduce(add, cmd, '')
112
113class execute(object):
114    """Execute commands or scripts. The 'output' is a funtion that handles the
115    output from the process. The 'input' is a function that blocks and returns
116    data to be written to stdin"""
117    def __init__(self, output = None, input = None, cleanup = None,
118                 error_prefix = '', verbose = False):
119        self.lock = threading.Lock()
120        self.output = output
121        self.input = input
122        self.cleanup = cleanup
123        self.error_prefix = error_prefix
124        self.verbose = verbose
125        self.shell_exe = None
126        self.shell_commands = False
127        self.path = None
128        self.environment = None
129        self.outputting = False
130        self.timing_out = False
131        self.proc = None
132
133    def capture(self, proc, command = 'pipe', timeout = None):
134        """Create 3 threads to read stdout and stderr and send to the output handler
135        and call an input handler is provided. Based on the 'communicate' code
136        in the subprocess module."""
137        def _writethread(exe, fh, input):
138            """Call the input handler and write it to the stdin. The input handler should
139            block and return None or False if this thread is to exit and True if this
140            is a timeout check."""
141            if trace_threads:
142                print('execute:_writethread: start')
143            encoding = True
144            try:
145                tmp = bytes('temp', sys.stdin.encoding)
146            except:
147                encoding = False
148            try:
149                while True:
150                    if trace_threads:
151                        print('execute:_writethread: call input', input)
152                    lines = input()
153                    if type(lines) == str or type(lines) == bytes:
154                        try:
155                            if encoding:
156                                lines = bytes(lines, sys.stdin.encoding)
157                            fh.write(lines)
158                        except:
159                            break
160                    if lines == None or \
161                       lines == False or \
162                       (lines == True and fh.closed):
163                        break
164            except:
165                if trace_threads:
166                    print('execute:_writethread: exception')
167                    print(traceback.format_exc())
168                pass
169            try:
170                fh.close()
171            except:
172                pass
173            if trace_threads:
174                print('execute:_writethread: finished')
175
176        def _readthread(exe, fh, out, prefix = ''):
177            """Read from a file handle and write to the output handler
178            until the file closes."""
179            def _output_line(line, exe, prefix, out, count):
180                #exe.lock.acquire()
181                #exe.outputting = True
182                #exe.lock.release()
183                if out:
184                    out(prefix + line)
185                else:
186                    log.output(prefix + line)
187                    if count > 10:
188                        log.flush()
189
190            if trace_threads:
191                print('execute:_readthread: start')
192            count = 0
193            line = ''
194            try:
195                while True:
196                    data = fh.read(1)
197                    if len(data) == 0:
198                        break
199                    # str and bytes are the same type in Python2
200                    if type(data) is not str and type(data) is bytes:
201                        data = data.decode(sys.stdout.encoding)
202                    for c in data:
203                        line += c
204                        if c == '\n':
205                            count += 1
206                            _output_line(line, exe, prefix, out, count)
207                            if count > 10:
208                                count = 0
209                            line = ''
210            except:
211                raise
212                if trace_threads:
213                    print('execute:_readthread: exception')
214                    print(traceback.format_exc())
215                pass
216            try:
217                fh.close()
218            except:
219                pass
220            if len(line):
221                _output_line(line, exe, prefix, out, 100)
222            if trace_threads:
223                print('execute:_readthread: finished')
224
225        def _timerthread(exe, interval, function):
226            """Timer thread is used to timeout a process if no output is
227            produced for the timeout interval."""
228            count = interval
229            while exe.timing_out:
230                time.sleep(1)
231                if count > 0:
232                    count -= 1
233                exe.lock.acquire()
234                if exe.outputting:
235                    count = interval
236                    exe.outputting = False
237                exe.lock.release()
238                if count == 0:
239                    try:
240                        proc.kill()
241                    except:
242                        pass
243                    else:
244                        function()
245                    break
246
247        name = os.path.basename(command[0])
248
249        stdin_thread = None
250        stdout_thread = None
251        stderr_thread = None
252        timeout_thread = None
253
254        if proc.stdout:
255            stdout_thread = threading.Thread(target = _readthread,
256                                             name = '_stdout[%s]' % (name),
257                                             args = (self,
258                                                     proc.stdout,
259                                                     self.output,
260                                                     ''))
261            stdout_thread.daemon = True
262            stdout_thread.start()
263        if proc.stderr:
264            stderr_thread = threading.Thread(target = _readthread,
265                                             name = '_stderr[%s]' % (name),
266                                             args = (self,
267                                                     proc.stderr,
268                                                     self.output,
269                                                     self.error_prefix))
270            stderr_thread.daemon = True
271            stderr_thread.start()
272        if self.input and proc.stdin:
273            stdin_thread = threading.Thread(target = _writethread,
274                                            name = '_stdin[%s]' % (name),
275                                            args = (self,
276                                                    proc.stdin,
277                                                    self.input))
278            stdin_thread.daemon = True
279            stdin_thread.start()
280        if timeout:
281            self.timing_out = True
282            timeout_thread = threading.Thread(target = _timerthread,
283                                              name = '_timeout[%s]' % (name),
284                                              args = (self,
285                                                      timeout[0],
286                                                      timeout[1]))
287            timeout_thread.daemon = True
288            timeout_thread.start()
289        try:
290            self.lock.acquire()
291            try:
292                self.proc = proc
293            except:
294                raise
295            finally:
296                self.lock.release()
297            exitcode = proc.wait()
298        except:
299            proc.kill()
300            raise
301        finally:
302            self.lock.acquire()
303            try:
304                self.proc = None
305            except:
306                raise
307            finally:
308                self.lock.release()
309            if self.cleanup:
310                self.cleanup(proc)
311            if timeout_thread:
312                self.timing_out = False
313                timeout_thread.join(10)
314            if stdin_thread:
315                stdin_thread.join(2)
316            if stdout_thread:
317                stdout_thread.join(2)
318            if stderr_thread:
319                stderr_thread.join(2)
320        return exitcode
321
322    def open(self, command, capture = True, shell = False,
323             cwd = None, env = None,
324             stdin = None, stdout = None, stderr = None,
325             timeout = None):
326        """Open a command with arguments. Provide the arguments as a list or
327        a string."""
328        if self.verbose:
329            s = command
330            if type(command) is list:
331                def add(x, y): return x + ' ' + str(y)
332                s = functools.reduce(add, command, '')[1:]
333            what = 'spawn'
334            if shell:
335                what = 'shell'
336            log.output(what + ': ' + s)
337        if self.output is None:
338            raise error.general('capture needs an output handler')
339        if shell and self.shell_exe:
340            command = arg_list(command)
341            command[:0] = self.shell_exe
342        if not stdin and self.input:
343            stdin = subprocess.PIPE
344        if not stdout:
345            stdout = subprocess.PIPE
346        if not stderr:
347            stderr = subprocess.PIPE
348        proc = None
349        if cwd is None:
350            cwd = self.path
351        if env is None:
352            env = self.environment
353        try:
354            # Work around a problem on Windows with commands that
355            # have a '.' and no extension. Windows needs the full
356            # command name.
357            if sys.platform == "win32" and type(command) is list:
358                if command[0].find('.') >= 0:
359                    r, e = os.path.splitext(command[0])
360                    if e not in ['.exe', '.com', '.bat']:
361                        command[0] = command[0] + '.exe'
362            log.trace('exe: %s' % (command))
363            proc = subprocess.Popen(command, shell = shell,
364                                    cwd = cwd, env = env,
365                                    stdin = stdin, stdout = stdout,
366                                    stderr = stderr)
367            if not capture:
368                return (0, proc)
369            if self.output is None:
370                raise error.general('capture needs an output handler')
371            exit_code = self.capture(proc, command, timeout)
372            if self.verbose:
373                log.output('exit: ' + str(exit_code))
374        except OSError as ose:
375            exit_code = ose.errno
376            if self.verbose:
377                log.output('exit: ' + str(ose))
378        return (exit_code, proc)
379
380    def spawn(self, command, capture = True, cwd = None, env = None,
381              stdin = None, stdout = None, stderr = None,
382              timeout = None):
383        """Spawn a command with arguments. Provide the arguments as a list or
384        a string."""
385        return self.open(command, capture, False, cwd, env,
386                         stdin, stdout, stderr, timeout)
387
388    def shell(self, command, capture = True, cwd = None, env = None,
389              stdin = None, stdout = None, stderr = None,
390              timeout = None):
391        """Execute a command within a shell context. The command can contain
392        argumments. The shell is specific to the operating system. For example
393        it is cmd.exe on Windows XP."""
394        return self.open(command, capture, True, cwd, env,
395                         stdin, stdout, stderr, timeout)
396
397    def command(self, command, args = None, capture = True, shell = False,
398                cwd = None, env = None,
399                stdin = None, stdout = None, stderr = None,
400                timeout = None):
401        """Run the command with the args. The args can be a list
402        or a string."""
403        if args and not type(args) is list:
404            args = arg_list(args)
405        cmd = [command]
406        if args:
407            cmd.extend(args)
408        return self.open(cmd, capture = capture, shell = shell,
409                         cwd = cwd, env = env,
410                         stdin = stdin, stdout = stdout, stderr = stderr,
411                         timeout = timeout)
412
413    def command_subst(self, command, substs, capture = True, shell = False,
414                      cwd = None, env = None,
415                      stdin = None, stdout = None, stderr = None,
416                      timeout = None):
417        """Run the command from the config data with the
418        option format string subsituted with the subst variables."""
419        args = arg_subst(command, substs)
420        return self.command(args[0], args[1:], capture = capture,
421                            shell = shell or self.shell_commands,
422                            cwd = cwd, env = env,
423                            stdin = stdin, stdout = stdout, stderr = stderr,
424                            timeout = timeout)
425
426    def set_shell(self, execute):
427        """Set the shell to execute when issuing a shell command."""
428        args = arg_list(execute)
429        if len(args) == 0 or not os.path.isfile(args[0]):
430            raise error.general('could find shell: ' + execute)
431        self.shell_exe = args
432
433    def command_use_shell(self):
434        """Force all commands to use a shell. This can be used with set_shell
435        to allow Unix commands be executed on Windows with a Unix shell such
436        as Cygwin or MSYS. This may cause piping to fail."""
437        self.shell_commands = True
438
439    def set_output(self, output):
440        """Set the output handler. The stdout of the last process in a pipe
441        line is passed to this handler."""
442        old_output = self.output
443        self.output = output
444        return old_output
445
446    def set_path(self, path):
447        """Set the path changed to before the child process is created."""
448        old_path = self.path
449        self.path = path
450        return old_path
451
452    def set_environ(self, environment):
453        """Set the environment passed to the child process when created."""
454        old_environment = self.environment
455        self.environment = environment
456        return old_environment
457
458    def kill(self):
459        self.lock.acquire()
460        try:
461            if self.proc is not None:
462                self.proc.kill()
463        except:
464            raise
465        finally:
466            self.lock.release()
467
468    def terminate(self):
469        self.lock.acquire()
470        try:
471            if self.proc is not None:
472                self.proc.terminate()
473        except:
474            raise
475        finally:
476            self.lock.release()
477
478    def send_signal(self, signal):
479        self.lock.acquire()
480        try:
481            if self.proc is not None:
482                print("sending sig")
483                self.proc.send_signal(signal)
484        except:
485            raise
486        finally:
487            self.lock.release()
488
489class capture_execution(execute):
490    """Capture all output as a string and return it."""
491
492    class _output_snapper:
493        def __init__(self, log = None, dump = False):
494            self.output = ''
495            self.log = log
496            self.dump = dump
497
498        def handler(self, text):
499            if not self.dump:
500                if self.log is not None:
501                    self.log.output(text)
502                else:
503                    self.output += text
504
505        def get_and_clear(self):
506            text = self.output
507            self.output = ''
508            return text.strip()
509
510    def __init__(self, log = None, dump = False, error_prefix = '', verbose = False):
511        self.snapper = capture_execution._output_snapper(log = log, dump = dump)
512        execute.__init__(self, output = self.snapper.handler,
513                         error_prefix = error_prefix,
514                         verbose = verbose)
515
516    def open(self, command, capture = True, shell = False, cwd = None, env = None,
517             stdin = None, stdout = None, stderr = None, timeout = None):
518        if not capture:
519            raise error.general('output capture must be true; leave as default')
520        #self.snapper.get_and_clear()
521        exit_code, proc = execute.open(self, command, capture = True, shell = shell,
522                                       cwd = cwd, env = env,
523                                       stdin = stdin, stdout = stdout, stderr = stderr,
524                                       timeout = timeout)
525        return (exit_code, proc, self.snapper.get_and_clear())
526
527    def set_output(self, output):
528        raise error.general('output capture cannot be overrided')
529
530if __name__ == "__main__":
531    def run_tests(e, commands, use_shell):
532        for c in commands['shell']:
533            e.shell(c)
534        for c in commands['spawn']:
535            e.spawn(c)
536        for c in commands['cmd']:
537            if type(c) is str:
538                e.command(c, shell = use_shell)
539            else:
540                e.command(c[0], c[1], shell = use_shell)
541        for c in commands['csubsts']:
542            e.command_subst(c[0], c[1], shell = use_shell)
543        ec, proc = e.command(commands['pipe'][0], commands['pipe'][1],
544                             capture = False, stdin = subprocess.PIPE)
545        if ec == 0:
546            print('piping input into ' + commands['pipe'][0] + ': ' + \
547                  commands['pipe'][2])
548            try:
549                out = bytes(commands['pipe'][2], sys.stdin.encoding)
550            except:
551                out = commands['pipe'][2]
552            proc.stdin.write(out)
553            proc.stdin.close()
554            e.capture(proc)
555            del proc
556
557    def capture_output(text):
558        print(text, end = '')
559
560    cmd_shell_test = 'if "%OS%" == "Windows_NT" (echo It is WinNT) else echo Is is not WinNT'
561    sh_shell_test = 'x="me"; if [ $x = "me" ]; then echo "It was me"; else "It was him"; fi'
562
563    commands = {}
564    commands['windows'] = {}
565    commands['unix'] = {}
566    commands['windows']['shell'] = ['cd', 'dir /w', '.\\xyz', cmd_shell_test]
567    commands['windows']['spawn'] = ['hostname', 'hostnameZZ', ['netstat', '/e']]
568    commands['windows']['cmd'] = [('ipconfig'), ('nslookup', 'www.python.org')]
569    commands['windows']['csubsts'] = [('netstat %0', ['-a']),
570                                      ('netstat %0 %1', ['-a', '-n'])]
571    commands['windows']['pipe'] = ('ftp', None, 'help\nquit')
572    commands['unix']['shell'] = ['pwd', 'ls -las', './xyz', sh_shell_test]
573    commands['unix']['spawn'] = ['ls', 'execute.pyc', ['ls', '-i']]
574    commands['unix']['cmd'] = [('date'), ('date', '-R'), ('date', ['-u', '+%d %D']),
575                               ('date', '-u "+%d %D %S"')]
576    commands['unix']['csubsts'] = [('date %0 "+%d %D %S"', ['-u']),
577                                   ('date %0 %1', ['-u', '+%d %D %S'])]
578    commands['unix']['pipe'] = ('grep', 'hello', 'hello world')
579
580    print(arg_list('cmd a1 a2 "a3 is a string" a4'))
581    print(arg_list('cmd b1 b2 "b3 is a string a4'))
582    print(arg_subst(['nothing', 'xx-%0-yyy', '%1', '%2-something'],
583                    ['subst0', 'subst1', 'subst2']))
584
585    e = execute(error_prefix = 'ERR: ', output = capture_output, verbose = True)
586    if sys.platform == "win32":
587        run_tests(e, commands['windows'], False)
588        if os.path.exists('c:\\msys\\1.0\\bin\\sh.exe'):
589            e.set_shell('c:\\msys\\1.0\\bin\\sh.exe --login -c')
590            commands['unix']['pipe'] = ('c:\\msys\\1.0\\bin\\grep',
591                                        'hello', 'hello world')
592            run_tests(e, commands['unix'], True)
593    else:
594        run_tests(e, commands['unix'], False)
595    del e
Note: See TracBrowser for help on using the repository browser.