source: rtems-tools/rtemstoolkit/execute.py @ a248471

5
Last change on this file since a248471 was 9e7ed07, checked in by Chris Johns <chrisj@…>, on 04/24/17 at 23:31:37

rtemstoolkit: Set proc buffering to 0 to not block on smaller reads.

  • Property mode set to 100755
File size: 23.0 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2017 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# 1. Redistributions of source code must retain the above copyright notice,
12# this list of conditions and the following disclaimer.
13#
14# 2. Redistributions in binary form must reproduce the above copyright notice,
15# this list of conditions and the following disclaimer in the documentation
16# and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31#
32# Execute commands or scripts.
33#
34# Note, the subprocess module is only in Python 2.4 or higher.
35#
36
37from __future__ import print_function
38
39import functools
40import io
41import os
42import re
43import sys
44import subprocess
45import threading
46import time
47import traceback
48
49#
50# Support to handle use in a package and as a unit test.
51# If there is a better way to let us know.
52#
53try:
54    from . import error
55    from . import log
56except (ValueError, SystemError):
57    import error
58    import log
59
60# Trace exceptions
61trace_threads = False
62
63# Redefine the PIPE from subprocess
64PIPE = subprocess.PIPE
65
66# Regular expression to find quotes.
67qstr = re.compile('[rR]?\'([^\\n\'\\\\]|\\\\.)*\'|[rR]?"([^\\n"\\\\]|\\\\.)*"')
68
69def check_type(command):
70    """Checks the type of command we have. The types are spawn and
71    shell."""
72    if command in ['spawn', 'shell']:
73        return True
74    return False
75
76def arg_list(args):
77    """Turn a string of arguments into a list suitable for
78    spawning a command. If the args are already a list return
79    it."""
80    if type(args) is list:
81        return args
82    argstr = args
83    args = []
84    while len(argstr):
85        qs = qstr.search(argstr)
86        if not qs:
87            args.extend(argstr.split())
88            argstr= ''
89        else:
90            # We have a quoted string. Get the string before
91            # the quoted string and splt on white space then
92            # add the quoted string as an option then remove
93            # the first + quoted string and try again
94            front = argstr[:qs.start()]
95            args.extend(front.split())
96            args.append(argstr[qs.start() + 1:qs.end() - 1])
97            argstr = argstr[qs.end():]
98    return args
99
100def arg_subst(command, substs):
101    """Substitute the %[0-9] in the command with the subst values."""
102    args = arg_list(command)
103    if substs:
104        for a in range(0, len(args)):
105            for r in range(0, len(substs)):
106                args[a] = re.compile(('%%%d' % (r))).sub(substs[r], args[a])
107    return args
108
109def arg_subst_str(command, subst):
110    cmd = arg_subst(command, subst)
111    def add(x, y): return x + ' ' + str(y)
112    return functools.reduce(add, cmd, '')
113
114class execute(object):
115    """Execute commands or scripts. The 'output' is a funtion that handles the
116    output from the process. The 'input' is a function that blocks and returns
117    data to be written to stdin"""
118    def __init__(self, output = None, input = None, cleanup = None,
119                 error_prefix = '', verbose = False):
120        self.lock = threading.Lock()
121        self.output = output
122        self.input = input
123        self.cleanup = cleanup
124        self.error_prefix = error_prefix
125        self.verbose = verbose
126        self.shell_exe = None
127        self.shell_commands = False
128        self.path = None
129        self.environment = None
130        self.outputting = False
131        self.timing_out = False
132        self.proc = None
133
134    def capture(self, proc, command = 'pipe', timeout = None):
135        """Create 3 threads to read stdout and stderr and send to the output handler
136        and call an input handler is provided. Based on the 'communicate' code
137        in the subprocess module."""
138        def _writethread(exe, fh, input):
139            """Call the input handler and write it to the stdin. The input handler should
140            block and return None or False if this thread is to exit and True if this
141            is a timeout check."""
142            if trace_threads:
143                print('execute:_writethread: start')
144            encoding = True
145            try:
146                tmp = bytes('temp', sys.stdin.encoding)
147            except:
148                encoding = False
149            try:
150                while True:
151                    if trace_threads:
152                        print('execute:_writethread: call input', input)
153                    lines = input()
154                    if type(lines) == str or type(lines) == bytes:
155                        try:
156                            if encoding:
157                                lines = bytes(lines, sys.stdin.encoding)
158                            fh.write(lines)
159                        except:
160                            break
161                    if lines == None or \
162                       lines == False or \
163                       (lines == True and fh.closed):
164                        break
165            except:
166                if trace_threads:
167                    print('execute:_writethread: exception')
168                    print(traceback.format_exc())
169                pass
170            try:
171                fh.close()
172            except:
173                pass
174            if trace_threads:
175                print('execute:_writethread: finished')
176
177        def _readthread(exe, fh, out, prefix = ''):
178            """Read from a file handle and write to the output handler
179            until the file closes."""
180            def _output_line(line, exe, prefix, out, count):
181                #exe.lock.acquire()
182                #exe.outputting = True
183                #exe.lock.release()
184                if out:
185                    out(prefix + line)
186                else:
187                    log.output(prefix + line)
188                    if count > 10:
189                        log.flush()
190
191            if trace_threads:
192                print('execute:_readthread: start')
193            count = 0
194            line = ''
195            try:
196                while True:
197                    #
198                    # The io module file handling return up to the size passed
199                    # in.
200                    #
201                    data = fh.read(4096)
202                    if len(data) == 0:
203                        break
204                    # str and bytes are the same type in Python2
205                    if type(data) is not str and type(data) is bytes:
206                        data = data.decode(sys.stdout.encoding)
207                    for c in data:
208                        line += c
209                        if c == '\n':
210                            count += 1
211                            _output_line(line, exe, prefix, out, count)
212                            if count > 10:
213                                count = 0
214                            line = ''
215            except:
216                raise
217                if trace_threads:
218                    print('execute:_readthread: exception')
219                    print(traceback.format_exc())
220                pass
221            try:
222                fh.close()
223            except:
224                pass
225            if len(line):
226                _output_line(line, exe, prefix, out, 100)
227            if trace_threads:
228                print('execute:_readthread: finished')
229
230        def _timerthread(exe, interval, function):
231            """Timer thread is used to timeout a process if no output is
232            produced for the timeout interval."""
233            count = interval
234            while exe.timing_out:
235                time.sleep(1)
236                if count > 0:
237                    count -= 1
238                exe.lock.acquire()
239                if exe.outputting:
240                    count = interval
241                    exe.outputting = False
242                exe.lock.release()
243                if count == 0:
244                    try:
245                        proc.kill()
246                    except:
247                        pass
248                    else:
249                        function()
250                    break
251
252        name = os.path.basename(command[0])
253
254        stdin_thread = None
255        stdout_thread = None
256        stderr_thread = None
257        timeout_thread = None
258
259        if proc.stdout:
260            stdout_thread = threading.Thread(target = _readthread,
261                                             name = '_stdout[%s]' % (name),
262                                             args = (self,
263                                                     io.open(proc.stdout.fileno(),
264                                                             mode = 'rb',
265                                                             buffering = 0,
266                                                             closefd = False),
267                                                     self.output,
268                                                     ''))
269            stdout_thread.daemon = True
270            stdout_thread.start()
271        if proc.stderr:
272            stderr_thread = threading.Thread(target = _readthread,
273                                             name = '_stderr[%s]' % (name),
274                                             args = (self,
275                                                     io.open(proc.stderr.fileno(),
276                                                             mode = 'rb',
277                                                             buffering = 0,
278                                                             closefd = False),
279                                                     self.output,
280                                                     self.error_prefix))
281            stderr_thread.daemon = True
282            stderr_thread.start()
283        if self.input and proc.stdin:
284            stdin_thread = threading.Thread(target = _writethread,
285                                            name = '_stdin[%s]' % (name),
286                                            args = (self,
287                                                    proc.stdin,
288                                                    self.input))
289            stdin_thread.daemon = True
290            stdin_thread.start()
291        if timeout:
292            self.timing_out = True
293            timeout_thread = threading.Thread(target = _timerthread,
294                                              name = '_timeout[%s]' % (name),
295                                              args = (self,
296                                                      timeout[0],
297                                                      timeout[1]))
298            timeout_thread.daemon = True
299            timeout_thread.start()
300        try:
301            self.lock.acquire()
302            try:
303                self.proc = proc
304            except:
305                raise
306            finally:
307                self.lock.release()
308            exitcode = proc.wait()
309        except:
310            proc.kill()
311            raise
312        finally:
313            self.lock.acquire()
314            try:
315                self.proc = None
316            except:
317                raise
318            finally:
319                self.lock.release()
320            if self.cleanup:
321                self.cleanup(proc)
322            if timeout_thread:
323                self.timing_out = False
324                timeout_thread.join(10)
325            if stdin_thread:
326                stdin_thread.join(2)
327            if stdout_thread:
328                stdout_thread.join(2)
329            if stderr_thread:
330                stderr_thread.join(2)
331        return exitcode
332
333    def open(self, command, capture = True, shell = False,
334             cwd = None, env = None,
335             stdin = None, stdout = None, stderr = None,
336             timeout = None):
337        """Open a command with arguments. Provide the arguments as a list or
338        a string."""
339        if self.verbose:
340            s = command
341            if type(command) is list:
342                def add(x, y): return x + ' ' + str(y)
343                s = functools.reduce(add, command, '')[1:]
344            what = 'spawn'
345            if shell:
346                what = 'shell'
347            log.output(what + ': ' + s)
348        if self.output is None:
349            raise error.general('capture needs an output handler')
350        if shell and self.shell_exe:
351            command = arg_list(command)
352            command[:0] = self.shell_exe
353        if not stdin and self.input:
354            stdin = subprocess.PIPE
355        if not stdout:
356            stdout = subprocess.PIPE
357        if not stderr:
358            stderr = subprocess.PIPE
359        proc = None
360        if cwd is None:
361            cwd = self.path
362        if env is None:
363            env = self.environment
364        try:
365            # Work around a problem on Windows with commands that
366            # have a '.' and no extension. Windows needs the full
367            # command name.
368            if sys.platform == "win32" and type(command) is list:
369                if command[0].find('.') >= 0:
370                    r, e = os.path.splitext(command[0])
371                    if e not in ['.exe', '.com', '.bat']:
372                        command[0] = command[0] + '.exe'
373            log.trace('exe: %s' % (command))
374            proc = subprocess.Popen(command, shell = shell,
375                                    cwd = cwd, env = env,
376                                    stdin = stdin, stdout = stdout,
377                                    stderr = stderr)
378            if not capture:
379                return (0, proc)
380            if self.output is None:
381                raise error.general('capture needs an output handler')
382            exit_code = self.capture(proc, command, timeout)
383            if self.verbose:
384                log.output('exit: ' + str(exit_code))
385        except OSError as ose:
386            exit_code = ose.errno
387            if self.verbose:
388                log.output('exit: ' + str(ose))
389        return (exit_code, proc)
390
391    def spawn(self, command, capture = True, cwd = None, env = None,
392              stdin = None, stdout = None, stderr = None,
393              timeout = None):
394        """Spawn a command with arguments. Provide the arguments as a list or
395        a string."""
396        return self.open(command, capture, False, cwd, env,
397                         stdin, stdout, stderr, timeout)
398
399    def shell(self, command, capture = True, cwd = None, env = None,
400              stdin = None, stdout = None, stderr = None,
401              timeout = None):
402        """Execute a command within a shell context. The command can contain
403        argumments. The shell is specific to the operating system. For example
404        it is cmd.exe on Windows XP."""
405        return self.open(command, capture, True, cwd, env,
406                         stdin, stdout, stderr, timeout)
407
408    def command(self, command, args = None, capture = True, shell = False,
409                cwd = None, env = None,
410                stdin = None, stdout = None, stderr = None,
411                timeout = None):
412        """Run the command with the args. The args can be a list
413        or a string."""
414        if args and not type(args) is list:
415            args = arg_list(args)
416        cmd = [command]
417        if args:
418            cmd.extend(args)
419        return self.open(cmd, capture = capture, shell = shell,
420                         cwd = cwd, env = env,
421                         stdin = stdin, stdout = stdout, stderr = stderr,
422                         timeout = timeout)
423
424    def command_subst(self, command, substs, capture = True, shell = False,
425                      cwd = None, env = None,
426                      stdin = None, stdout = None, stderr = None,
427                      timeout = None):
428        """Run the command from the config data with the
429        option format string subsituted with the subst variables."""
430        args = arg_subst(command, substs)
431        return self.command(args[0], args[1:], capture = capture,
432                            shell = shell or self.shell_commands,
433                            cwd = cwd, env = env,
434                            stdin = stdin, stdout = stdout, stderr = stderr,
435                            timeout = timeout)
436
437    def set_shell(self, execute):
438        """Set the shell to execute when issuing a shell command."""
439        args = arg_list(execute)
440        if len(args) == 0 or not os.path.isfile(args[0]):
441            raise error.general('could find shell: ' + execute)
442        self.shell_exe = args
443
444    def command_use_shell(self):
445        """Force all commands to use a shell. This can be used with set_shell
446        to allow Unix commands be executed on Windows with a Unix shell such
447        as Cygwin or MSYS. This may cause piping to fail."""
448        self.shell_commands = True
449
450    def set_output(self, output):
451        """Set the output handler. The stdout of the last process in a pipe
452        line is passed to this handler."""
453        old_output = self.output
454        self.output = output
455        return old_output
456
457    def set_path(self, path):
458        """Set the path changed to before the child process is created."""
459        old_path = self.path
460        self.path = path
461        return old_path
462
463    def set_environ(self, environment):
464        """Set the environment passed to the child process when created."""
465        old_environment = self.environment
466        self.environment = environment
467        return old_environment
468
469    def kill(self):
470        self.lock.acquire()
471        try:
472            if self.proc is not None:
473                self.proc.kill()
474        except:
475            raise
476        finally:
477            self.lock.release()
478
479    def terminate(self):
480        self.lock.acquire()
481        try:
482            if self.proc is not None:
483                self.proc.terminate()
484        except:
485            raise
486        finally:
487            self.lock.release()
488
489    def send_signal(self, signal):
490        self.lock.acquire()
491        try:
492            if self.proc is not None:
493                print("sending sig")
494                self.proc.send_signal(signal)
495        except:
496            raise
497        finally:
498            self.lock.release()
499
500class capture_execution(execute):
501    """Capture all output as a string and return it."""
502
503    class _output_snapper:
504        def __init__(self, log = None, dump = False):
505            self.output = ''
506            self.log = log
507            self.dump = dump
508
509        def handler(self, text):
510            if not self.dump:
511                if self.log is not None:
512                    self.log.output(text)
513                else:
514                    self.output += text
515
516        def get_and_clear(self):
517            text = self.output
518            self.output = ''
519            return text.strip()
520
521    def __init__(self, log = None, dump = False, error_prefix = '', verbose = False):
522        self.snapper = capture_execution._output_snapper(log = log, dump = dump)
523        execute.__init__(self, output = self.snapper.handler,
524                         error_prefix = error_prefix,
525                         verbose = verbose)
526
527    def open(self, command, capture = True, shell = False, cwd = None, env = None,
528             stdin = None, stdout = None, stderr = None, timeout = None):
529        if not capture:
530            raise error.general('output capture must be true; leave as default')
531        #self.snapper.get_and_clear()
532        exit_code, proc = execute.open(self, command, capture = True, shell = shell,
533                                       cwd = cwd, env = env,
534                                       stdin = stdin, stdout = stdout, stderr = stderr,
535                                       timeout = timeout)
536        return (exit_code, proc, self.snapper.get_and_clear())
537
538    def set_output(self, output):
539        raise error.general('output capture cannot be overrided')
540
541if __name__ == "__main__":
542    def run_tests(e, commands, use_shell):
543        for c in commands['shell']:
544            e.shell(c)
545        for c in commands['spawn']:
546            e.spawn(c)
547        for c in commands['cmd']:
548            if type(c) is str:
549                e.command(c, shell = use_shell)
550            else:
551                e.command(c[0], c[1], shell = use_shell)
552        for c in commands['csubsts']:
553            e.command_subst(c[0], c[1], shell = use_shell)
554        ec, proc = e.command(commands['pipe'][0], commands['pipe'][1],
555                             capture = False, stdin = subprocess.PIPE)
556        if ec == 0:
557            print('piping input into ' + commands['pipe'][0] + ': ' + \
558                  commands['pipe'][2])
559            try:
560                out = bytes(commands['pipe'][2], sys.stdin.encoding)
561            except:
562                out = commands['pipe'][2]
563            proc.stdin.write(out)
564            proc.stdin.close()
565            e.capture(proc)
566            del proc
567
568    def capture_output(text):
569        print(text, end = '')
570
571    cmd_shell_test = 'if "%OS%" == "Windows_NT" (echo It is WinNT) else echo Is is not WinNT'
572    sh_shell_test = 'x="me"; if [ $x = "me" ]; then echo "It was me"; else "It was him"; fi'
573
574    commands = {}
575    commands['windows'] = {}
576    commands['unix'] = {}
577    commands['windows']['shell'] = ['cd', 'dir /w', '.\\xyz', cmd_shell_test]
578    commands['windows']['spawn'] = ['hostname', 'hostnameZZ', ['netstat', '/e']]
579    commands['windows']['cmd'] = [('ipconfig'), ('nslookup', 'www.python.org')]
580    commands['windows']['csubsts'] = [('netstat %0', ['-a']),
581                                      ('netstat %0 %1', ['-a', '-n'])]
582    commands['windows']['pipe'] = ('ftp', None, 'help\nquit')
583    commands['unix']['shell'] = ['pwd', 'ls -las', './xyz', sh_shell_test]
584    commands['unix']['spawn'] = ['ls', 'execute.pyc', ['ls', '-i']]
585    commands['unix']['cmd'] = [('date'), ('date', '-R'), ('date', ['-u', '+%d %D']),
586                               ('date', '-u "+%d %D %S"')]
587    commands['unix']['csubsts'] = [('date %0 "+%d %D %S"', ['-u']),
588                                   ('date %0 %1', ['-u', '+%d %D %S'])]
589    commands['unix']['pipe'] = ('grep', 'hello', 'hello world')
590
591    print(arg_list('cmd a1 a2 "a3 is a string" a4'))
592    print(arg_list('cmd b1 b2 "b3 is a string a4'))
593    print(arg_subst(['nothing', 'xx-%0-yyy', '%1', '%2-something'],
594                    ['subst0', 'subst1', 'subst2']))
595
596    e = execute(error_prefix = 'ERR: ', output = capture_output, verbose = True)
597    if sys.platform == "win32":
598        run_tests(e, commands['windows'], False)
599        if os.path.exists('c:\\msys\\1.0\\bin\\sh.exe'):
600            e.set_shell('c:\\msys\\1.0\\bin\\sh.exe --login -c')
601            commands['unix']['pipe'] = ('c:\\msys\\1.0\\bin\\grep',
602                                        'hello', 'hello world')
603            run_tests(e, commands['unix'], True)
604    else:
605        run_tests(e, commands['unix'], False)
606    del e
Note: See TracBrowser for help on using the repository browser.