source: rtems-tools/rtemstoolkit/execute.py @ 2de37f3

4.105
Last change on this file since 2de37f3 was 2de37f3, checked in by Chris Johns <chrisj@…>, on 03/09/16 at 03:27:42

Python 2 and python 3 refactor fixes.

Updates #2619.
  • Property mode set to 100755
File size: 22.2 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2016 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Redistribution and use in source and binary forms, with or without
9# modification, are permitted provided that the following conditions are met:
10#
11# 1. Redistributions of source code must retain the above copyright notice,
12# this list of conditions and the following disclaimer.
13#
14# 2. Redistributions in binary form must reproduce the above copyright notice,
15# this list of conditions and the following disclaimer in the documentation
16# and/or other materials provided with the distribution.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28# POSSIBILITY OF SUCH DAMAGE.
29#
30
31#
32# Execute commands or scripts.
33#
34# Note, the subprocess module is only in Python 2.4 or higher.
35#
36
37from __future__ import print_function
38
39import functools
40import os
41import re
42import sys
43import subprocess
44import threading
45import time
46
47#
48# Support to handle use in a package and as a unit test.
49# If there is a better way to let us know.
50#
51try:
52    from . import error
53    from . import log
54except (ValueError, SystemError):
55    import error
56    import log
57
58# Trace exceptions
59trace_threads = False
60
61# Redefine the PIPE from subprocess
62PIPE = subprocess.PIPE
63
64# Regular expression to find quotes.
65qstr = re.compile('[rR]?\'([^\\n\'\\\\]|\\\\.)*\'|[rR]?"([^\\n"\\\\]|\\\\.)*"')
66
67def check_type(command):
68    """Checks the type of command we have. The types are spawn and
69    shell."""
70    if command in ['spawn', 'shell']:
71        return True
72    return False
73
74def arg_list(args):
75    """Turn a string of arguments into a list suitable for
76    spawning a command. If the args are already a list return
77    it."""
78    if type(args) is list:
79        return args
80    argstr = args
81    args = []
82    while len(argstr):
83        qs = qstr.search(argstr)
84        if not qs:
85            args.extend(argstr.split())
86            argstr= ''
87        else:
88            # We have a quoted string. Get the string before
89            # the quoted string and splt on white space then
90            # add the quoted string as an option then remove
91            # the first + quoted string and try again
92            front = argstr[:qs.start()]
93            args.extend(front.split())
94            args.append(argstr[qs.start() + 1:qs.end() - 1])
95            argstr = argstr[qs.end():]
96    return args
97
98def arg_subst(command, substs):
99    """Substitute the %[0-9] in the command with the subst values."""
100    args = arg_list(command)
101    if substs:
102        for a in range(0, len(args)):
103            for r in range(0, len(substs)):
104                args[a] = re.compile(('%%%d' % (r))).sub(substs[r], args[a])
105    return args
106
107def arg_subst_str(command, subst):
108    cmd = arg_subst(command, subst)
109    def add(x, y): return x + ' ' + str(y)
110    return functools.reduce(add, cmd, '')
111
112class execute(object):
113    """Execute commands or scripts. The 'output' is a funtion that handles the
114    output from the process. The 'input' is a function that blocks and returns
115    data to be written to stdin"""
116    def __init__(self, output = None, input = None, cleanup = None,
117                 error_prefix = '', verbose = False):
118        self.lock = threading.Lock()
119        self.output = output
120        self.input = input
121        self.cleanup = cleanup
122        self.error_prefix = error_prefix
123        self.verbose = verbose
124        self.shell_exe = None
125        self.shell_commands = False
126        self.path = None
127        self.environment = None
128        self.outputting = False
129        self.timing_out = False
130        self.proc = None
131
132    def capture(self, proc, command = 'pipe', timeout = None):
133        """Create 3 threads to read stdout and stderr and send to the output handler
134        and call an input handler is provided. Based on the 'communicate' code
135        in the subprocess module."""
136        def _writethread(exe, fh, input):
137            """Call the input handler and write it to the stdin. The input handler should
138            block and return None or False if this thread is to exit and True if this
139            is a timeout check."""
140            if trace_threads:
141                print('executte:_writethread: start')
142            encoding = True
143            try:
144                tmp = bytes('temp', sys.stdin.encoding)
145            except:
146                encoding = False
147            try:
148                while True:
149                    lines = eval(input())
150                    if type(lines) == str or type(lines) == bytes:
151                        try:
152                            if encoding:
153                                lines = bytes(lines, sys.stdin.encoding)
154                            fh.write(lines)
155                        except:
156                            break
157                    if lines == None or \
158                       lines == False or \
159                       (lines == True and fh.closed):
160                        break
161            except:
162                if trace_threads:
163                    print('executte:_writethread: exception')
164                pass
165            try:
166                fh.close()
167            except:
168                pass
169            if trace_threads:
170                print('executte:_writethread: finished')
171
172        def _readthread(exe, fh, out, prefix = ''):
173            """Read from a file handle and write to the output handler
174            until the file closes."""
175            def _output_line(line, exe, prefix, out, count):
176                #exe.lock.acquire()
177                #exe.outputting = True
178                #exe.lock.release()
179                if out:
180                    out(prefix + line)
181                else:
182                    log.output(prefix + line)
183                    if count > 10:
184                        log.flush()
185
186            if trace_threads:
187                print('executte:_readthread: start')
188            count = 0
189            line = ''
190            try:
191                while True:
192                    data = fh.read(1)
193                    if len(data) == 0:
194                        break
195                    # str and bytes are the same type in Python2
196                    if type(data) is not str and type(data) is bytes:
197                        data = data.decode(sys.stdout.encoding)
198                    for c in data:
199                        line += c
200                        if c == '\n':
201                            count += 1
202                            _output_line(line, exe, prefix, out, count)
203                            if count > 10:
204                                count = 0
205                            line = ''
206            except:
207                raise
208                if trace_threads:
209                    print('executte:_readthread: exception')
210                pass
211            try:
212                fh.close()
213            except:
214                pass
215            if len(line):
216                _output_line(line, exe, prefix, out, 100)
217            if trace_threads:
218                print('executte:_readthread: finished')
219
220        def _timerthread(exe, interval, function):
221            """Timer thread is used to timeout a process if no output is
222            produced for the timeout interval."""
223            count = interval
224            while exe.timing_out:
225                time.sleep(1)
226                if count > 0:
227                    count -= 1
228                exe.lock.acquire()
229                if exe.outputting:
230                    count = interval
231                    exe.outputting = False
232                exe.lock.release()
233                if count == 0:
234                    try:
235                        proc.kill()
236                    except:
237                        pass
238                    else:
239                        function()
240                    break
241
242        name = os.path.basename(command[0])
243
244        stdin_thread = None
245        stdout_thread = None
246        stderr_thread = None
247        timeout_thread = None
248
249        if proc.stdout:
250            stdout_thread = threading.Thread(target = _readthread,
251                                             name = '_stdout[%s]' % (name),
252                                             args = (self,
253                                                     proc.stdout,
254                                                     self.output,
255                                                     ''))
256            stdout_thread.daemon = True
257            stdout_thread.start()
258        if proc.stderr:
259            stderr_thread = threading.Thread(target = _readthread,
260                                             name = '_stderr[%s]' % (name),
261                                             args = (self,
262                                                     proc.stderr,
263                                                     self.output,
264                                                     self.error_prefix))
265            stderr_thread.daemon = True
266            stderr_thread.start()
267        if self.input and proc.stdin:
268            stdin_thread = threading.Thread(target = _writethread,
269                                            name = '_stdin[%s]' % (name),
270                                            args = (self,
271                                                    proc.stdin,
272                                                    self.input))
273            stdin_thread.daemon = True
274            stdin_thread.start()
275        if timeout:
276            self.timing_out = True
277            timeout_thread = threading.Thread(target = _timerthread,
278                                              name = '_timeout[%s]' % (name),
279                                              args = (self,
280                                                      timeout[0],
281                                                      timeout[1]))
282            timeout_thread.daemon = True
283            timeout_thread.start()
284        try:
285            self.lock.acquire()
286            try:
287                self.proc = proc
288            except:
289                raise
290            finally:
291                self.lock.release()
292            exitcode = proc.wait()
293        except:
294            proc.kill()
295            raise
296        finally:
297            self.lock.acquire()
298            try:
299                self.proc = None
300            except:
301                raise
302            finally:
303                self.lock.release()
304            if self.cleanup:
305                self.cleanup(proc)
306            if timeout_thread:
307                self.timing_out = False
308                timeout_thread.join(10)
309            if stdin_thread:
310                stdin_thread.join(2)
311            if stdout_thread:
312                stdout_thread.join(2)
313            if stderr_thread:
314                stderr_thread.join(2)
315        return exitcode
316
317    def open(self, command, capture = True, shell = False,
318             cwd = None, env = None,
319             stdin = None, stdout = None, stderr = None,
320             timeout = None):
321        """Open a command with arguments. Provide the arguments as a list or
322        a string."""
323        if self.verbose:
324            s = command
325            if type(command) is list:
326                def add(x, y): return x + ' ' + str(y)
327                s = functools.reduce(add, command, '')[1:]
328            what = 'spawn'
329            if shell:
330                what = 'shell'
331            log.output(what + ': ' + s)
332        if self.output is None:
333            raise error.general('capture needs an output handler')
334        if shell and self.shell_exe:
335            command = arg_list(command)
336            command[:0] = self.shell_exe
337        if not stdin and self.input:
338            stdin = subprocess.PIPE
339        if not stdout:
340            stdout = subprocess.PIPE
341        if not stderr:
342            stderr = subprocess.PIPE
343        proc = None
344        if cwd is None:
345            cwd = self.path
346        if env is None:
347            env = self.environment
348        try:
349            # Work around a problem on Windows with commands that
350            # have a '.' and no extension. Windows needs the full
351            # command name.
352            if sys.platform == "win32" and type(command) is list:
353                if command[0].find('.') >= 0:
354                    r, e = os.path.splitext(command[0])
355                    if e not in ['.exe', '.com', '.bat']:
356                        command[0] = command[0] + '.exe'
357            log.trace('exe: %s' % (command))
358            proc = subprocess.Popen(command, shell = shell,
359                                    cwd = cwd, env = env,
360                                    stdin = stdin, stdout = stdout,
361                                    stderr = stderr)
362            if not capture:
363                return (0, proc)
364            if self.output is None:
365                raise error.general('capture needs an output handler')
366            exit_code = self.capture(proc, command, timeout)
367            if self.verbose:
368                log.output('exit: ' + str(exit_code))
369        except OSError as ose:
370            exit_code = ose.errno
371            if self.verbose:
372                log.output('exit: ' + str(ose))
373        return (exit_code, proc)
374
375    def spawn(self, command, capture = True, cwd = None, env = None,
376              stdin = None, stdout = None, stderr = None,
377              timeout = None):
378        """Spawn a command with arguments. Provide the arguments as a list or
379        a string."""
380        return self.open(command, capture, False, cwd, env,
381                         stdin, stdout, stderr, timeout)
382
383    def shell(self, command, capture = True, cwd = None, env = None,
384              stdin = None, stdout = None, stderr = None,
385              timeout = None):
386        """Execute a command within a shell context. The command can contain
387        argumments. The shell is specific to the operating system. For example
388        it is cmd.exe on Windows XP."""
389        return self.open(command, capture, True, cwd, env,
390                         stdin, stdout, stderr, timeout)
391
392    def command(self, command, args = None, capture = True, shell = False,
393                cwd = None, env = None,
394                stdin = None, stdout = None, stderr = None,
395                timeout = None):
396        """Run the command with the args. The args can be a list
397        or a string."""
398        if args and not type(args) is list:
399            args = arg_list(args)
400        cmd = [command]
401        if args:
402            cmd.extend(args)
403        return self.open(cmd, capture = capture, shell = shell,
404                         cwd = cwd, env = env,
405                         stdin = stdin, stdout = stdout, stderr = stderr,
406                         timeout = timeout)
407
408    def command_subst(self, command, substs, capture = True, shell = False,
409                      cwd = None, env = None,
410                      stdin = None, stdout = None, stderr = None,
411                      timeout = None):
412        """Run the command from the config data with the
413        option format string subsituted with the subst variables."""
414        args = arg_subst(command, substs)
415        return self.command(args[0], args[1:], capture = capture,
416                            shell = shell or self.shell_commands,
417                            cwd = cwd, env = env,
418                            stdin = stdin, stdout = stdout, stderr = stderr,
419                            timeout = timeout)
420
421    def set_shell(self, execute):
422        """Set the shell to execute when issuing a shell command."""
423        args = arg_list(execute)
424        if len(args) == 0 or not os.path.isfile(args[0]):
425            raise error.general('could find shell: ' + execute)
426        self.shell_exe = args
427
428    def command_use_shell(self):
429        """Force all commands to use a shell. This can be used with set_shell
430        to allow Unix commands be executed on Windows with a Unix shell such
431        as Cygwin or MSYS. This may cause piping to fail."""
432        self.shell_commands = True
433
434    def set_output(self, output):
435        """Set the output handler. The stdout of the last process in a pipe
436        line is passed to this handler."""
437        old_output = self.output
438        self.output = output
439        return old_output
440
441    def set_path(self, path):
442        """Set the path changed to before the child process is created."""
443        old_path = self.path
444        self.path = path
445        return old_path
446
447    def set_environ(self, environment):
448        """Set the environment passed to the child process when created."""
449        old_environment = self.environment
450        self.environment = environment
451        return old_environment
452
453    def kill(self):
454        self.lock.acquire()
455        try:
456            if self.proc is not None:
457                self.proc.kill()
458        except:
459            raise
460        finally:
461            self.lock.release()
462
463    def terminate(self):
464        self.lock.acquire()
465        try:
466            if self.proc is not None:
467                self.proc.terminate()
468        except:
469            raise
470        finally:
471            self.lock.release()
472
473    def send_signal(self, signal):
474        self.lock.acquire()
475        try:
476            if self.proc is not None:
477                print("sending sig")
478                self.proc.send_signal(signal)
479        except:
480            raise
481        finally:
482            self.lock.release()
483
484class capture_execution(execute):
485    """Capture all output as a string and return it."""
486
487    class _output_snapper:
488        def __init__(self, log = None, dump = False):
489            self.output = ''
490            self.log = log
491            self.dump = dump
492
493        def handler(self, text):
494            if not self.dump:
495                if self.log is not None:
496                    self.log.output(text)
497                else:
498                    self.output += text
499
500        def get_and_clear(self):
501            text = self.output
502            self.output = ''
503            return text.strip()
504
505    def __init__(self, log = None, dump = False, error_prefix = '', verbose = False):
506        self.snapper = capture_execution._output_snapper(log = log, dump = dump)
507        execute.__init__(self, output = self.snapper.handler,
508                         error_prefix = error_prefix,
509                         verbose = verbose)
510
511    def open(self, command, capture = True, shell = False, cwd = None, env = None,
512             stdin = None, stdout = None, stderr = None, timeout = None):
513        if not capture:
514            raise error.general('output capture must be true; leave as default')
515        #self.snapper.get_and_clear()
516        exit_code, proc = execute.open(self, command, capture = True, shell = shell,
517                                       cwd = cwd, env = env,
518                                       stdin = stdin, stdout = stdout, stderr = stderr,
519                                       timeout = timeout)
520        return (exit_code, proc, self.snapper.get_and_clear())
521
522    def set_output(self, output):
523        raise error.general('output capture cannot be overrided')
524
525if __name__ == "__main__":
526    def run_tests(e, commands, use_shell):
527        for c in commands['shell']:
528            e.shell(c)
529        for c in commands['spawn']:
530            e.spawn(c)
531        for c in commands['cmd']:
532            if type(c) is str:
533                e.command(c, shell = use_shell)
534            else:
535                e.command(c[0], c[1], shell = use_shell)
536        for c in commands['csubsts']:
537            e.command_subst(c[0], c[1], shell = use_shell)
538        ec, proc = e.command(commands['pipe'][0], commands['pipe'][1],
539                             capture = False, stdin = subprocess.PIPE)
540        if ec == 0:
541            print('piping input into ' + commands['pipe'][0] + ': ' + \
542                  commands['pipe'][2])
543            try:
544                out = bytes(commands['pipe'][2], sys.stdin.encoding)
545            except:
546                out = commands['pipe'][2]
547            proc.stdin.write(out)
548            proc.stdin.close()
549            e.capture(proc)
550            del proc
551
552    def capture_output(text):
553        print(text, end = '')
554
555    cmd_shell_test = 'if "%OS%" == "Windows_NT" (echo It is WinNT) else echo Is is not WinNT'
556    sh_shell_test = 'x="me"; if [ $x = "me" ]; then echo "It was me"; else "It was him"; fi'
557
558    commands = {}
559    commands['windows'] = {}
560    commands['unix'] = {}
561    commands['windows']['shell'] = ['cd', 'dir /w', '.\\xyz', cmd_shell_test]
562    commands['windows']['spawn'] = ['hostname', 'hostnameZZ', ['netstat', '/e']]
563    commands['windows']['cmd'] = [('ipconfig'), ('nslookup', 'www.python.org')]
564    commands['windows']['csubsts'] = [('netstat %0', ['-a']),
565                                      ('netstat %0 %1', ['-a', '-n'])]
566    commands['windows']['pipe'] = ('ftp', None, 'help\nquit')
567    commands['unix']['shell'] = ['pwd', 'ls -las', './xyz', sh_shell_test]
568    commands['unix']['spawn'] = ['ls', 'execute.pyc', ['ls', '-i']]
569    commands['unix']['cmd'] = [('date'), ('date', '-R'), ('date', ['-u', '+%d %D']),
570                               ('date', '-u "+%d %D %S"')]
571    commands['unix']['csubsts'] = [('date %0 "+%d %D %S"', ['-u']),
572                                   ('date %0 %1', ['-u', '+%d %D %S'])]
573    commands['unix']['pipe'] = ('grep', 'hello', 'hello world')
574
575    print(arg_list('cmd a1 a2 "a3 is a string" a4'))
576    print(arg_list('cmd b1 b2 "b3 is a string a4'))
577    print(arg_subst(['nothing', 'xx-%0-yyy', '%1', '%2-something'],
578                    ['subst0', 'subst1', 'subst2']))
579
580    e = execute(error_prefix = 'ERR: ', output = capture_output, verbose = True)
581    if sys.platform == "win32":
582        run_tests(e, commands['windows'], False)
583        if os.path.exists('c:\\msys\\1.0\\bin\\sh.exe'):
584            e.set_shell('c:\\msys\\1.0\\bin\\sh.exe --login -c')
585            commands['unix']['pipe'] = ('c:\\msys\\1.0\\bin\\grep',
586                                        'hello', 'hello world')
587            run_tests(e, commands['unix'], True)
588    else:
589        run_tests(e, commands['unix'], False)
590    del e
Note: See TracBrowser for help on using the repository browser.