source: rtems-source-builder/source-builder/sb/execute.py @ 76188ee4

4.11
Last change on this file since 76188ee4 was 5142bec, checked in by Chris Johns <chrisj@…>, on 04/21/13 at 08:37:02

Refactor the logging support.
  • Property mode set to 100755
File size: 14.5 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2012 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# Execute commands or scripts.
22#
23# Note, the subprocess module is only in Python 2.4 or higher.
24#
25
26import os
27import re
28import sys
29import subprocess
30import threading
31
32import error
33import log
34
35# Redefine the PIPE from subprocess
36PIPE = subprocess.PIPE
37
38# Regular expression to find quotes.
39qstr = re.compile('[rR]?\'([^\\n\'\\\\]|\\\\.)*\'|[rR]?"([^\\n"\\\\]|\\\\.)*"')
40
41def check_type(command):
42    """Checks the type of command we have. The types are spawn and
43    shell."""
44    if command in ['spawn', 'shell']:
45        return True
46    return False
47
48def arg_list(args):
49    """Turn a string of arguments into a list suitable for
50    spawning a command. If the args are already a list return
51    it."""
52    if type(args) is list:
53        return args
54    argstr = args
55    args = []
56    while len(argstr):
57        qs = qstr.search(argstr)
58        if not qs:
59            args.extend(argstr.split())
60            argstr= ''
61        else:
62            # We have a quoted string. Get the string before
63            # the quoted string and splt on white space then
64            # add the quoted string as an option then remove
65            # the first + quoted string and try again
66            front = argstr[:qs.start()]
67            args.extend(front.split())
68            args.append(argstr[qs.start() + 1:qs.end() - 1])
69            argstr = argstr[qs.end():]
70    return args
71
72def arg_subst(command, substs):
73    """Substitute the %[0-9] in the command with the subst values."""
74    args = arg_list(command)
75    if substs:
76        for a in range(0, len(args)):
77            for r in range(0, len(substs)):
78                args[a] = re.compile(('%%%d' % (r))).sub(substs[r], args[a])
79    return args
80
81def arg_subst_str(command, subst):
82    cmd = arg_subst(command, subst)
83    def add(x, y): return x + ' ' + str(y)
84    return reduce(add, cmd, '')
85
86class execute:
87    """Execute commands or scripts. The 'output' is a funtion
88    that handles the output from the process."""
89    def __init__(self, output = None, error_prefix = '', verbose = False):
90        self.output = output
91        self.error_prefix = error_prefix
92        self.verbose = verbose
93        self.shell_exe = None
94        self.shell_commands = False
95        self.path = None
96        self.environment = None
97
98    def capture(self, proc, timeout = None):
99        """Create 2 threads to read stdout and stderr and send to the
100        output handler. Based on the 'communicate' code in the subprocess
101        module."""
102        def _readthread(fh, out, prefix = ''):
103            """Read from a file handle and write to the output handler
104            until the file closes."""
105            count = 0
106            while True:
107                line = fh.readline()
108                count += 1
109                if len(line) == 0:
110                    break
111                if out:
112                    out(prefix + line)
113                else:
114                    log.output(prefix + line)
115                    if count > 10:
116                        log.flush()
117                        count = 0
118
119        def _timerthread(proc, timer):
120            """Timer thread calls the timer handler if one
121            is present once a second. The user provides a handler
122            and returns False to kill the process or True continue."""
123            while True:
124                time.sleep(1)
125                if not timer(proc):
126                    proc.stdout.close()
127                    proc.stderr.close()
128
129        if proc.stdout:
130            stdout_thread = threading.Thread(target = _readthread,
131                                             args = (proc.stdout,
132                                                     self.output,
133                                                     ''))
134            stdout_thread.setDaemon(True)
135            stdout_thread.start()
136        if proc.stderr:
137            stderr_thread = threading.Thread(target = _readthread,
138                                             args = (proc.stderr,
139                                                     self.output,
140                                                     self.error_prefix))
141            stderr_thread.setDaemon(True)
142            stderr_thread.start()
143        if proc.stdout:
144            stdout_thread.join()
145        if proc.stderr:
146            stderr_thread.join()
147        return proc.wait()
148
149    def open(self, command, capture = True, shell = False,
150             cwd = None, env = None,
151             stdin = None, stdout = None, stderr = None):
152        """Open a command with arguments. Provide the arguments as a list or
153        a string."""
154        if self.verbose:
155            s = command
156            if type(command) is list:
157                def add(x, y): return x + ' ' + str(y)
158                s = reduce(add, command, '')[1:]
159            what = 'spawn'
160            if shell:
161                what = 'shell'
162            log.output(what + ': ' + s)
163        if shell and self.shell_exe:
164            command = arg_list(command)
165            command[:0] = self.shell_exe
166        if not stdout:
167            stdout = subprocess.PIPE
168        if not stderr:
169            stderr = subprocess.PIPE
170        proc = None
171        if cwd is None:
172            cwd = self.path
173        if env is None:
174            env = self.environment
175        try:
176            # Work around a problem on Windows with commands that
177            # have a '.' and no extension. Windows needs the full
178            # command name.
179            if sys.platform == "win32" and type(command) is list:
180                if command[0].find('.') >= 0:
181                    r, e = os.path.splitext(command[0])
182                    if e not in ['.exe', '.com', '.bat']:
183                        command[0] = command[0] + '.exe'
184            log.trace('exe: %s' % (command))
185            proc = subprocess.Popen(command, shell = shell,
186                                    cwd = cwd, env = env,
187                                    stdin = stdin, stdout = stdout,
188                                    stderr = stderr)
189            if not capture:
190                return (0, proc)
191            exit_code = self.capture(proc)
192            if self.verbose:
193                log.output('exit: ' + str(exit_code))
194        except OSError, ose:
195            exit_code = ose.errno
196            if self.verbose:
197                log.output('exit: ' + str(ose))
198        return (exit_code, proc)
199
200    def spawn(self, command, capture = True, cwd = None, env = None,
201              stdin = None, stdout = None, stderr = None):
202        """Spawn a command with arguments. Provide the arguments as a list or
203        a string."""
204        return self.open(command, capture, False, cwd, env,
205                         stdin, stdout, stderr)
206
207    def shell(self, command, capture = True, cwd = None, env = None,
208              stdin = None, stdout = None, stderr = None):
209        """Execute a command within a shell context. The command can contain
210        argumments. The shell is specific to the operating system. For example
211        it is cmd.exe on Windows XP."""
212        return self.open(command, capture, True, cwd, env,
213                         stdin, stdout, stderr)
214
215    def command(self, command, args = None, capture = True, shell = False,
216                cwd = None, env = None,
217                stdin = None, stdout = None, stderr = None):
218        """Run the command with the args. The args can be a list
219        or a string."""
220        if args and not type(args) is list:
221            args = arg_list(args)
222        cmd = [command]
223        if args:
224            cmd.extend(args)
225        return self.open(cmd, capture = capture, shell = shell,
226                         cwd = cwd, env = env,
227                         stdin = stdin, stdout = stdout, stderr = stderr)
228
229    def command_subst(self, command, substs, capture = True, shell = False,
230                      cwd = None, env = None,
231                      stdin = None, stdout = None, stderr = None):
232        """Run the command from the config data with the
233        option format string subsituted with the subst variables."""
234        args = arg_subst(command, substs)
235        return self.command(args[0], args[1:], capture = capture,
236                            shell = shell or self.shell_commands,
237                            cwd = cwd, env = env,
238                            stdin = stdin, stdout = stdout, stderr = stderr)
239
240    def set_shell(self, execute):
241        """Set the shell to execute when issuing a shell command."""
242        args = arg_list(execute)
243        if len(args) == 0 or not os.path.isfile(args[0]):
244            raise error.general('could find shell: ' + execute)
245        self.shell_exe = args
246
247    def command_use_shell(self):
248        """Force all commands to use a shell. This can be used with set_shell
249        to allow Unix commands be executed on Windows with a Unix shell such
250        as Cygwin or MSYS. This may cause piping to fail."""
251        self.shell_commands = True
252
253    def set_output(self, output):
254        """Set the output handler. The stdout of the last process in a pipe
255        line is passed to this handler."""
256        old_output = self.output
257        self.output = output
258        return old_output
259
260    def set_path(self, path):
261        """Set the path changed to before the child process is created."""
262        old_path = self.path
263        self.path = path
264        return old_path
265
266    def set_environ(self, environment):
267        """Set the environment passed to the child process when created."""
268        old_environment = self.environment
269        self.environment = environment
270        return old_environment
271
272class capture_execution(execute):
273    """Capture all output as a string and return it."""
274
275    class _output_snapper:
276        def __init__(self, log = None, dump = False):
277            self.output = ''
278            self.log = log
279            self.dump = dump
280
281        def handler(self, text):
282            if not self.dump:
283                if self.log is not None:
284                    self.log.output(text)
285                else:
286                    self.output += text
287
288        def get_and_clear(self):
289            text = self.output
290            self.output = ''
291            return text.strip()
292
293    def __init__(self, log = None, dump = False, error_prefix = '', verbose = False):
294        self.snapper = capture_execution._output_snapper(log = log, dump = dump)
295        execute.__init__(self, output = self.snapper.handler,
296                         error_prefix = error_prefix,
297                         verbose = verbose)
298
299    def open(self, command, capture = True, shell = False, cwd = None, env = None,
300             stdin = None, stdout = None, stderr = None):
301        if not capture:
302            raise error.general('output capture must be true; leave as default')
303        #self.snapper.get_and_clear()
304        exit_code, proc = execute.open(self, command, capture = True, shell = shell,
305                                       cwd = cwd, env = env,
306                                       stdin = stdin, stdout = stdout, stderr = stderr)
307        return (exit_code, proc, self.snapper.get_and_clear())
308
309    def set_output(self, output):
310        raise error.general('output capture cannot be overrided')
311
312if __name__ == "__main__":
313    def run_tests(e, commands, use_shell):
314        for c in commands['shell']:
315            e.shell(c)
316        for c in commands['spawn']:
317            e.spawn(c)
318        for c in commands['cmd']:
319            if type(c) is str:
320                e.command(c, shell = use_shell)
321            else:
322                e.command(c[0], c[1], shell = use_shell)
323        for c in commands['csubsts']:
324            e.command_subst(c[0], c[1], shell = use_shell)
325        ec, proc = e.command(commands['pipe'][0], commands['pipe'][1],
326                             capture = False, stdin = subprocess.PIPE)
327        if ec == 0:
328            print 'piping input into ' + commands['pipe'][0] + ': ' + \
329                  commands['pipe'][2]
330            proc.stdin.write(commands['pipe'][2])
331            proc.stdin.close()
332            e.capture(proc)
333            del proc
334
335    cmd_shell_test = 'if "%OS%" == "Windows_NT" (echo It is WinNT) else echo Is is not WinNT'
336    sh_shell_test = 'x="me"; if [ $x = "me" ]; then echo "It was me"; else "It was him"; fi'
337
338    commands = {}
339    commands['windows'] = {}
340    commands['unix'] = {}
341    commands['windows']['shell'] = ['cd', 'dir /w', '.\\xyz', cmd_shell_test]
342    commands['windows']['spawn'] = ['hostname', 'hostnameZZ', ['netstat', '/e']]
343    commands['windows']['cmd'] = [('ipconfig'), ('nslookup', 'www.python.org')]
344    commands['windows']['csubsts'] = [('netstat %0', ['-a']),
345                                      ('netstat %0 %1', ['-a', '-n'])]
346    commands['windows']['pipe'] = ('ftp', None, 'help\nquit')
347    commands['unix']['shell'] = ['pwd', 'ls -las', './xyz', sh_shell_test]
348    commands['unix']['spawn'] = ['ls', 'execute.pyc', ['ls', '-i']]
349    commands['unix']['cmd'] = [('date'), ('date', '-R'), ('date', ['-u', '+%d %D']),
350                               ('date', '-u "+%d %D %S"')]
351    commands['unix']['csubsts'] = [('date %0 "+%d %D %S"', ['-u']),
352                                   ('date %0 %1', ['-u', '+%d %D %S'])]
353    commands['unix']['pipe'] = ('grep', 'hello', 'hello world')
354
355    print arg_list('cmd a1 a2 "a3 is a string" a4')
356    print arg_list('cmd b1 b2 "b3 is a string a4')
357    print arg_subst(['nothing', 'xx-%0-yyy', '%1', '%2-something'],
358                    ['subst0', 'subst1', 'subst2'])
359
360    e = execute(error_prefix = 'ERR: ', verbose = True)
361    if sys.platform == "win32":
362        run_tests(e, commands['windows'], False)
363        if os.path.exists('c:\\msys\\1.0\\bin\\sh.exe'):
364            e.set_shell('c:\\msys\\1.0\\bin\\sh.exe --login -c')
365            commands['unix']['pipe'] = ('c:\\msys\\1.0\\bin\\grep',
366                                        'hello', 'hello world')
367            run_tests(e, commands['unix'], True)
368    else:
369        run_tests(e, commands['unix'], False)
370    del e
Note: See TracBrowser for help on using the repository browser.