source: rtems-tools/rtemstoolkit/execute.py @ 3e9b179

4.11
Last change on this file since 3e9b179 was 3e9b179, checked in by Chris Johns <chrisj@…>, on 03/14/16 at 04:09:00

rtemstoolkit: Fix execute's writer thread to not eval() the input.

The conversion to Python3 added an eval() call which is wrong.

Fix the spelling in execute.

Fix labels in the tester gdb locking.

Check the debug-trace arguments.

Close #2642.
  • Property mode set to 100755
File size: 21.8 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2016 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# 1. Redistributions of source code must retain the above copyright notice,
12# this list of conditions and the following disclaimer.
13#
14# 2. Redistributions in binary form must reproduce the above copyright notice,
15# this list of conditions and the following disclaimer in the documentation
16# and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31#
32# Execute commands or scripts.
33#
34# Note, the subprocess module is only in Python 2.4 or higher.
35#
36
37from __future__ import print_function
38
39import functools
40import os
41import re
42import sys
43import subprocess
44import threading
45import time
46
47#
48# Support to handle use in a package and as a unit test.
49# If there is a better way to let us know.
50#
51try:
52    from . import error
53    from . import log
54except (ValueError, SystemError):
55    import error
56    import log
57
58# Trace exceptions
59trace_threads = False
60
61# Redefine the PIPE from subprocess
62PIPE = subprocess.PIPE
63
64# Regular expression to find quotes.
65qstr = re.compile('[rR]?\'([^\\n\'\\\\]|\\\\.)*\'|[rR]?"([^\\n"\\\\]|\\\\.)*"')
66
67def check_type(command):
68    """Checks the type of command we have. The types are spawn and
69    shell."""
70    if command in ['spawn', 'shell']:
71        return True
72    return False
73
74def arg_list(args):
75    """Turn a string of arguments into a list suitable for
76    spawning a command. If the args are already a list return
77    it."""
78    if type(args) is list:
79        return args
80    argstr = args
81    args = []
82    while len(argstr):
83        qs = qstr.search(argstr)
84        if not qs:
85            args.extend(argstr.split())
86            argstr= ''
87        else:
88            # We have a quoted string. Get the string before
89            # the quoted string and splt on white space then
90            # add the quoted string as an option then remove
91            # the first + quoted string and try again
92            front = argstr[:qs.start()]
93            args.extend(front.split())
94            args.append(argstr[qs.start() + 1:qs.end() - 1])
95            argstr = argstr[qs.end():]
96    return args
97
98def arg_subst(command, substs):
99    """Substitute the %[0-9] in the command with the subst values."""
100    args = arg_list(command)
101    if substs:
102        for a in range(0, len(args)):
103            for r in range(0, len(substs)):
104                args[a] = re.compile(('%%%d' % (r))).sub(substs[r], args[a])
105    return args
106
107def arg_subst_str(command, subst):
108    cmd = arg_subst(command, subst)
109    def add(x, y): return x + ' ' + str(y)
110    return functools.reduce(add, cmd, '')
111
112class execute(object):
113    """Execute commands or scripts. The 'output' is a funtion that handles the
114    output from the process. The 'input' is a function that blocks and returns
115    data to be written to stdin"""
116    def __init__(self, output = None, input = None, cleanup = None,
117                 error_prefix = '', verbose = False):
118        self.lock = threading.Lock()
119        self.output = output
120        self.input = input
121        self.cleanup = cleanup
122        self.error_prefix = error_prefix
123        self.verbose = verbose
124        self.shell_exe = None
125        self.shell_commands = False
126        self.path = None
127        self.environment = None
128        self.outputting = False
129        self.timing_out = False
130        self.proc = None
131
132    def capture(self, proc, command = 'pipe', timeout = None):
133        """Create 3 threads to read stdout and stderr and send to the output handler
134        and call an input handler is provided. Based on the 'communicate' code
135        in the subprocess module."""
136        def _writethread(exe, fh, input):
137            """Call the input handler and write it to the stdin. The input handler should
138            block and return None or False if this thread is to exit and True if this
139            is a timeout check."""
140            if trace_threads:
141                print('execute:_writethread: start')
142            try:
143                while True:
144                    lines = input()
145                    if type(lines) == str:
146                        try:
147                            fh.write(bytes(lines, sys.stdin.encoding))
148                        except:
149                            break
150                    if lines == None or \
151                       lines == False or \
152                       (lines == True and fh.closed):
153                        break
154            except:
155                if trace_threads:
156                    print('execute:_writethread: exception')
157                pass
158            try:
159                fh.close()
160            except:
161                pass
162            if trace_threads:
163                print('execute:_writethread: finished')
164
165        def _readthread(exe, fh, out, prefix = ''):
166            """Read from a file handle and write to the output handler
167            until the file closes."""
168            def _output_line(line, exe, prefix, out, count):
169                #exe.lock.acquire()
170                #exe.outputting = True
171                #exe.lock.release()
172                if out:
173                    out(prefix + line)
174                else:
175                    log.output(prefix + line)
176                    if count > 10:
177                        log.flush()
178
179            if trace_threads:
180                print('execute:_readthread: start')
181            count = 0
182            line = ''
183            try:
184                while True:
185                    data = fh.read(1)
186                    if len(data) == 0:
187                        break
188                    # str and bytes are the same type in Python2
189                    if type(data) is not str and type(data) is bytes:
190                        data = data.decode(sys.stdout.encoding)
191                    for c in data:
192                        line += c
193                        if c == '\n':
194                            count += 1
195                            _output_line(line, exe, prefix, out, count)
196                            if count > 10:
197                                count = 0
198                            line = ''
199            except:
200                raise
201                if trace_threads:
202                    print('execute:_readthread: exception')
203                pass
204            try:
205                fh.close()
206            except:
207                pass
208            if len(line):
209                _output_line(line, exe, prefix, out, 100)
210            if trace_threads:
211                print('execute:_readthread: finished')
212
213        def _timerthread(exe, interval, function):
214            """Timer thread is used to timeout a process if no output is
215            produced for the timeout interval."""
216            count = interval
217            while exe.timing_out:
218                time.sleep(1)
219                if count > 0:
220                    count -= 1
221                exe.lock.acquire()
222                if exe.outputting:
223                    count = interval
224                    exe.outputting = False
225                exe.lock.release()
226                if count == 0:
227                    try:
228                        proc.kill()
229                    except:
230                        pass
231                    else:
232                        function()
233                    break
234
235        name = os.path.basename(command[0])
236
237        stdin_thread = None
238        stdout_thread = None
239        stderr_thread = None
240        timeout_thread = None
241
242        if proc.stdout:
243            stdout_thread = threading.Thread(target = _readthread,
244                                             name = '_stdout[%s]' % (name),
245                                             args = (self,
246                                                     proc.stdout,
247                                                     self.output,
248                                                     ''))
249            stdout_thread.daemon = True
250            stdout_thread.start()
251        if proc.stderr:
252            stderr_thread = threading.Thread(target = _readthread,
253                                             name = '_stderr[%s]' % (name),
254                                             args = (self,
255                                                     proc.stderr,
256                                                     self.output,
257                                                     self.error_prefix))
258            stderr_thread.daemon = True
259            stderr_thread.start()
260        if self.input and proc.stdin:
261            stdin_thread = threading.Thread(target = _writethread,
262                                            name = '_stdin[%s]' % (name),
263                                            args = (self,
264                                                    proc.stdin,
265                                                    self.input))
266            stdin_thread.daemon = True
267            stdin_thread.start()
268        if timeout:
269            self.timing_out = True
270            timeout_thread = threading.Thread(target = _timerthread,
271                                              name = '_timeout[%s]' % (name),
272                                              args = (self,
273                                                      timeout[0],
274                                                      timeout[1]))
275            timeout_thread.daemon = True
276            timeout_thread.start()
277        try:
278            self.lock.acquire()
279            try:
280                self.proc = proc
281            except:
282                raise
283            finally:
284                self.lock.release()
285            exitcode = proc.wait()
286        except:
287            proc.kill()
288            raise
289        finally:
290            self.lock.acquire()
291            try:
292                self.proc = None
293            except:
294                raise
295            finally:
296                self.lock.release()
297            if self.cleanup:
298                self.cleanup(proc)
299            if timeout_thread:
300                self.timing_out = False
301                timeout_thread.join(10)
302            if stdin_thread:
303                stdin_thread.join(2)
304            if stdout_thread:
305                stdout_thread.join(2)
306            if stderr_thread:
307                stderr_thread.join(2)
308        return exitcode
309
310    def open(self, command, capture = True, shell = False,
311             cwd = None, env = None,
312             stdin = None, stdout = None, stderr = None,
313             timeout = None):
314        """Open a command with arguments. Provide the arguments as a list or
315        a string."""
316        if self.verbose:
317            s = command
318            if type(command) is list:
319                def add(x, y): return x + ' ' + str(y)
320                s = functools.reduce(add, command, '')[1:]
321            what = 'spawn'
322            if shell:
323                what = 'shell'
324            log.output(what + ': ' + s)
325        if self.output is None:
326            raise error.general('capture needs an output handler')
327        if shell and self.shell_exe:
328            command = arg_list(command)
329            command[:0] = self.shell_exe
330        if not stdin and self.input:
331            stdin = subprocess.PIPE
332        if not stdout:
333            stdout = subprocess.PIPE
334        if not stderr:
335            stderr = subprocess.PIPE
336        proc = None
337        if cwd is None:
338            cwd = self.path
339        if env is None:
340            env = self.environment
341        try:
342            # Work around a problem on Windows with commands that
343            # have a '.' and no extension. Windows needs the full
344            # command name.
345            if sys.platform == "win32" and type(command) is list:
346                if command[0].find('.') >= 0:
347                    r, e = os.path.splitext(command[0])
348                    if e not in ['.exe', '.com', '.bat']:
349                        command[0] = command[0] + '.exe'
350            log.trace('exe: %s' % (command))
351            proc = subprocess.Popen(command, shell = shell,
352                                    cwd = cwd, env = env,
353                                    stdin = stdin, stdout = stdout,
354                                    stderr = stderr)
355            if not capture:
356                return (0, proc)
357            exit_code = self.capture(proc, command, timeout)
358            if self.verbose:
359                log.output('exit: ' + str(exit_code))
360        except OSError as ose:
361            exit_code = ose.errno
362            if self.verbose:
363                log.output('exit: ' + str(ose))
364        return (exit_code, proc)
365
366    def spawn(self, command, capture = True, cwd = None, env = None,
367              stdin = None, stdout = None, stderr = None,
368              timeout = None):
369        """Spawn a command with arguments. Provide the arguments as a list or
370        a string."""
371        return self.open(command, capture, False, cwd, env,
372                         stdin, stdout, stderr, timeout)
373
374    def shell(self, command, capture = True, cwd = None, env = None,
375              stdin = None, stdout = None, stderr = None,
376              timeout = None):
377        """Execute a command within a shell context. The command can contain
378        argumments. The shell is specific to the operating system. For example
379        it is cmd.exe on Windows XP."""
380        return self.open(command, capture, True, cwd, env,
381                         stdin, stdout, stderr, timeout)
382
383    def command(self, command, args = None, capture = True, shell = False,
384                cwd = None, env = None,
385                stdin = None, stdout = None, stderr = None,
386                timeout = None):
387        """Run the command with the args. The args can be a list
388        or a string."""
389        if args and not type(args) is list:
390            args = arg_list(args)
391        cmd = [command]
392        if args:
393            cmd.extend(args)
394        return self.open(cmd, capture = capture, shell = shell,
395                         cwd = cwd, env = env,
396                         stdin = stdin, stdout = stdout, stderr = stderr,
397                         timeout = timeout)
398
399    def command_subst(self, command, substs, capture = True, shell = False,
400                      cwd = None, env = None,
401                      stdin = None, stdout = None, stderr = None,
402                      timeout = None):
403        """Run the command from the config data with the
404        option format string subsituted with the subst variables."""
405        args = arg_subst(command, substs)
406        return self.command(args[0], args[1:], capture = capture,
407                            shell = shell or self.shell_commands,
408                            cwd = cwd, env = env,
409                            stdin = stdin, stdout = stdout, stderr = stderr,
410                            timeout = timeout)
411
412    def set_shell(self, execute):
413        """Set the shell to execute when issuing a shell command."""
414        args = arg_list(execute)
415        if len(args) == 0 or not os.path.isfile(args[0]):
416            raise error.general('could find shell: ' + execute)
417        self.shell_exe = args
418
419    def command_use_shell(self):
420        """Force all commands to use a shell. This can be used with set_shell
421        to allow Unix commands be executed on Windows with a Unix shell such
422        as Cygwin or MSYS. This may cause piping to fail."""
423        self.shell_commands = True
424
425    def set_output(self, output):
426        """Set the output handler. The stdout of the last process in a pipe
427        line is passed to this handler."""
428        old_output = self.output
429        self.output = output
430        return old_output
431
432    def set_path(self, path):
433        """Set the path changed to before the child process is created."""
434        old_path = self.path
435        self.path = path
436        return old_path
437
438    def set_environ(self, environment):
439        """Set the environment passed to the child process when created."""
440        old_environment = self.environment
441        self.environment = environment
442        return old_environment
443
444    def kill(self):
445        self.lock.acquire()
446        try:
447            if self.proc is not None:
448                self.proc.kill()
449        except:
450            raise
451        finally:
452            self.lock.release()
453
454    def terminate(self):
455        self.lock.acquire()
456        try:
457            if self.proc is not None:
458                self.proc.terminate()
459        except:
460            raise
461        finally:
462            self.lock.release()
463
464    def send_signal(self, signal):
465        self.lock.acquire()
466        try:
467            if self.proc is not None:
468                print("sending sig")
469                self.proc.send_signal(signal)
470        except:
471            raise
472        finally:
473            self.lock.release()
474
475class capture_execution(execute):
476    """Capture all output as a string and return it."""
477
478    class _output_snapper:
479        def __init__(self, log = None, dump = False):
480            self.output = ''
481            self.log = log
482            self.dump = dump
483
484        def handler(self, text):
485            if not self.dump:
486                if self.log is not None:
487                    self.log.output(text)
488                else:
489                    self.output += text
490
491        def get_and_clear(self):
492            text = self.output
493            self.output = ''
494            return text.strip()
495
496    def __init__(self, log = None, dump = False, error_prefix = '', verbose = False):
497        self.snapper = capture_execution._output_snapper(log = log, dump = dump)
498        execute.__init__(self, output = self.snapper.handler,
499                         error_prefix = error_prefix,
500                         verbose = verbose)
501
502    def open(self, command, capture = True, shell = False, cwd = None, env = None,
503             stdin = None, stdout = None, stderr = None, timeout = None):
504        if not capture:
505            raise error.general('output capture must be true; leave as default')
506        #self.snapper.get_and_clear()
507        exit_code, proc = execute.open(self, command, capture = True, shell = shell,
508                                       cwd = cwd, env = env,
509                                       stdin = stdin, stdout = stdout, stderr = stderr,
510                                       timeout = timeout)
511        return (exit_code, proc, self.snapper.get_and_clear())
512
513    def set_output(self, output):
514        raise error.general('output capture cannot be overrided')
515
516if __name__ == "__main__":
517
518    def run_tests(e, commands, use_shell):
519        for c in commands['shell']:
520            e.shell(c)
521        for c in commands['spawn']:
522            e.spawn(c)
523        for c in commands['cmd']:
524            if type(c) is str:
525                e.command(c, shell = use_shell)
526            else:
527                e.command(c[0], c[1], shell = use_shell)
528        for c in commands['csubsts']:
529            e.command_subst(c[0], c[1], shell = use_shell)
530        ec, proc = e.command(commands['pipe'][0], commands['pipe'][1],
531                             capture = False, stdin = subprocess.PIPE)
532        if ec == 0:
533            print('piping input into ' + commands['pipe'][0] + ': ' + \
534                  commands['pipe'][2])
535            try:
536                out = bytes(commands['pipe'][2], sys.stdin.encoding)
537            except:
538                out = bytes(commands['pipe'][2])
539            proc.stdin.write(out)
540            proc.stdin.close()
541            e.capture(proc)
542            del proc
543
544    def capture_output(text):
545        print(text, end = '')
546
547    cmd_shell_test = 'if "%OS%" == "Windows_NT" (echo It is WinNT) else echo Is is not WinNT'
548    sh_shell_test = 'x="me"; if [ $x = "me" ]; then echo "It was me"; else "It was him"; fi'
549
550    commands = {}
551    commands['windows'] = {}
552    commands['unix'] = {}
553    commands['windows']['shell'] = ['cd', 'dir /w', '.\\xyz', cmd_shell_test]
554    commands['windows']['spawn'] = ['hostname', 'hostnameZZ', ['netstat', '/e']]
555    commands['windows']['cmd'] = [('ipconfig'), ('nslookup', 'www.python.org')]
556    commands['windows']['csubsts'] = [('netstat %0', ['-a']),
557                                      ('netstat %0 %1', ['-a', '-n'])]
558    commands['windows']['pipe'] = ('ftp', None, 'help\nquit')
559    commands['unix']['shell'] = ['pwd', 'ls -las', './xyz', sh_shell_test]
560    commands['unix']['spawn'] = ['ls', 'execute.pyc', ['ls', '-i']]
561    commands['unix']['cmd'] = [('date'), ('date', '-R'), ('date', ['-u', '+%d %D']),
562                               ('date', '-u "+%d %D %S"')]
563    commands['unix']['csubsts'] = [('date %0 "+%d %D %S"', ['-u']),
564                                   ('date %0 %1', ['-u', '+%d %D %S'])]
565    commands['unix']['pipe'] = ('grep', 'hello', 'hello world')
566
567    print(arg_list('cmd a1 a2 "a3 is a string" a4'))
568    print(arg_list('cmd b1 b2 "b3 is a string a4'))
569    print(arg_subst(['nothing', 'xx-%0-yyy', '%1', '%2-something'],
570                    ['subst0', 'subst1', 'subst2']))
571
572    e = execute(error_prefix = 'ERR: ', output = capture_output, verbose = True)
573    if sys.platform == "win32":
574        run_tests(e, commands['windows'], False)
575        if os.path.exists('c:\\msys\\1.0\\bin\\sh.exe'):
576            e.set_shell('c:\\msys\\1.0\\bin\\sh.exe --login -c')
577            commands['unix']['pipe'] = ('c:\\msys\\1.0\\bin\\grep',
578                                        'hello', 'hello world')
579            run_tests(e, commands['unix'], True)
580    else:
581        run_tests(e, commands['unix'], False)
582    del e
Note: See TracBrowser for help on using the repository browser.