source: rtems-source-builder/source-builder/sb/execute.py @ c80560d

4.104.114.95
Last change on this file since c80560d was c80560d, checked in by Chris Johns <chrisj@…>, on 11/05/12 at 23:09:40

Move into the source-builder tree.
  • Property mode set to 100755
File size: 14.3 KB
Line 
1#
2# RTEMS Tools Project (http://www.rtems.org/)
3# Copyright 2010-2012 Chris Johns (chrisj@rtems.org)
4# All rights reserved.
5#
6# This file is part of the RTEMS Tools package in 'rtems-tools'.
7#
8# Permission to use, copy, modify, and/or distribute this software for any
9# purpose with or without fee is hereby granted, provided that the above
10# copyright notice and this permission notice appear in all copies.
11#
12# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
20#
21# Execute commands or scripts.
22#
23# Note, the subprocess module is only in Python 2.4 or higher.
24#
25
26import os
27import re
28import sys
29import subprocess
30import threading
31
32import error
33import log
34
35# Redefine the PIPE from subprocess
36PIPE = subprocess.PIPE
37
38# Regular expression to find quotes.
39qstr = re.compile('[rR]?\'([^\\n\'\\\\]|\\\\.)*\'|[rR]?"([^\\n"\\\\]|\\\\.)*"')
40
41def check_type(command):
42    """Checks the type of command we have. The types are spawn and
43    shell."""
44    if command in ['spawn', 'shell']:
45        return True
46    return False
47
48def arg_list(args):
49    """Turn a string of arguments into a list suitable for
50    spawning a command. If the args are already a list return
51    it."""
52    if type(args) is list:
53        return args
54    argstr = args
55    args = []
56    while len(argstr):
57        qs = qstr.search(argstr)
58        if not qs:
59            args.extend(argstr.split())
60            argstr= ''
61        else:
62            # We have a quoted string. Get the string before
63            # the quoted string and splt on white space then
64            # add the quoted string as an option then remove
65            # the first + quoted string and try again
66            front = argstr[:qs.start()]
67            args.extend(front.split())
68            args.append(argstr[qs.start() + 1:qs.end() - 1])
69            argstr = argstr[qs.end():]
70    return args
71
72def arg_subst(command, substs):
73    """Substitute the %[0-9] in the command with the subst values."""
74    args = arg_list(command)
75    if substs:
76        for a in range(0, len(args)):
77            for r in range(0, len(substs)):
78                args[a] = re.compile(('%%%d' % (r))).sub(substs[r], args[a])
79    return args
80
81def arg_subst_str(command, subst):
82    cmd = arg_subst(command, subst)
83    def add(x, y): return x + ' ' + str(y)
84    return reduce(add, cmd, '')
85
86class execute:
87    """Execute commands or scripts. The 'output' is a funtion
88    that handles the output from the process."""
89    def __init__(self, output = None, error_prefix = '', verbose = False):
90        self.output = output
91        self.error_prefix = error_prefix
92        self.verbose = verbose
93        self.shell_exe = None
94        self.shell_commands = False
95        self.path = None
96        self.environment = None
97
98    def capture(self, proc, timeout = None):
99        """Create 2 threads to read stdout and stderr and send to the
100        output handler. Based on the 'communicate' code in the subprocess
101        module."""
102        def _readthread(fh, out, prefix = ''):
103            """Read from a file handle and write to the output handler
104            until the file closes."""
105            while True:
106                line = fh.readline()
107                if len(line) == 0:
108                    break
109                if out:
110                    out(prefix + line)
111                else:
112                    log.output(prefix + line)
113        def _timerthread(proc, timer):
114            """Timer thread calls the timer handler if one
115            is present once a second. The user provides a handler
116            and returns False to kill the process or True continue."""
117            while True:
118                time.sleep(1)
119                if not timer(proc):
120                    proc.stdout.close()
121                    proc.stderr.close()
122
123        if proc.stdout:
124            stdout_thread = threading.Thread(target = _readthread,
125                                             args = (proc.stdout,
126                                                     self.output,
127                                                     ''))
128            stdout_thread.setDaemon(True)
129            stdout_thread.start()
130        if proc.stderr:
131            stderr_thread = threading.Thread(target = _readthread,
132                                             args = (proc.stderr,
133                                                     self.output,
134                                                     self.error_prefix))
135            stderr_thread.setDaemon(True)
136            stderr_thread.start()
137        if proc.stdout:
138            stdout_thread.join()
139        if proc.stderr:
140            stderr_thread.join()
141        return proc.wait()
142
143    def open(self, command, capture = True, shell = False,
144             cwd = None, env = None,
145             stdin = None, stdout = None, stderr = None):
146        """Open a command with arguments. Provide the arguments as a list or
147        a string."""
148        if self.verbose:
149            s = command
150            if type(command) is list:
151                def add(x, y): return x + ' ' + str(y)
152                s = reduce(add, command, '')[1:]
153            what = 'spawn'
154            if shell:
155                what = 'shell'
156            log.output(what + ': ' + s)
157        if shell and self.shell_exe:
158            command = arg_list(command)
159            command[:0] = self.shell_exe
160        if not stdout:
161            stdout = subprocess.PIPE
162        if not stderr:
163            stderr = subprocess.PIPE
164        proc = None
165        if cwd is None:
166            cwd = self.path
167        if env is None:
168            env = self.environment
169        try:
170            # Work around a problem on Windows with commands that
171            # have a '.' and no extension. Windows needs the full
172            # command name.
173            if sys.platform == "win32" and type(command) is list:
174                if command[0].find('.') >= 0:
175                    r, e = os.path.splitext(command[0])
176                    if e not in ['.exe', '.com', '.bat']:
177                        command[0] = command[0] + '.exe'
178            proc = subprocess.Popen(command, shell = shell,
179                                    cwd = cwd, env = env,
180                                    stdin = stdin, stdout = stdout,
181                                    stderr = stderr)
182            if not capture:
183                return (0, proc)
184            exit_code = self.capture(proc)
185            if self.verbose:
186                log.output('exit: ' + str(exit_code))
187        except OSError, ose:
188            exit_code = ose.errno
189            if self.verbose:
190                log.output('exit: ' + str(ose))
191        return (exit_code, proc)
192
193    def spawn(self, command, capture = True, cwd = None, env = None,
194              stdin = None, stdout = None, stderr = None):
195        """Spawn a command with arguments. Provide the arguments as a list or
196        a string."""
197        return self.open(command, capture, False, cwd, env,
198                         stdin, stdout, stderr)
199
200    def shell(self, command, capture = True, cwd = None, env = None,
201              stdin = None, stdout = None, stderr = None):
202        """Execute a command within a shell context. The command can contain
203        argumments. The shell is specific to the operating system. For example
204        it is cmd.exe on Windows XP."""
205        return self.open(command, capture, True, cwd, env,
206                         stdin, stdout, stderr)
207
208    def command(self, command, args = None, capture = True, shell = False,
209                cwd = None, env = None,
210                stdin = None, stdout = None, stderr = None):
211        """Run the command with the args. The args can be a list
212        or a string."""
213        if args and not type(args) is list:
214            args = arg_list(args)
215        cmd = [command]
216        if args:
217            cmd.extend(args)
218        return self.open(cmd, capture = capture, shell = shell,
219                         cwd = cwd, env = env,
220                         stdin = stdin, stdout = stdout, stderr = stderr)
221
222    def command_subst(self, command, substs, capture = True, shell = False,
223                      cwd = None, env = None,
224                      stdin = None, stdout = None, stderr = None):
225        """Run the command from the config data with the
226        option format string subsituted with the subst variables."""
227        args = arg_subst(command, substs)
228        return self.command(args[0], args[1:], capture = capture,
229                            shell = shell or self.shell_commands,
230                            cwd = cwd, env = env,
231                            stdin = stdin, stdout = stdout, stderr = stderr)
232
233    def set_shell(self, execute):
234        """Set the shell to execute when issuing a shell command."""
235        args = arg_list(execute)
236        if len(args) == 0 or not os.path.isfile(args[0]):
237            raise error.general('could find shell: ' + execute)
238        self.shell_exe = args
239
240    def command_use_shell(self):
241        """Force all commands to use a shell. This can be used with set_shell
242        to allow Unix commands be executed on Windows with a Unix shell such
243        as Cygwin or MSYS. This may cause piping to fail."""
244        self.shell_commands = True
245
246    def set_output(self, output):
247        """Set the output handler. The stdout of the last process in a pipe
248        line is passed to this handler."""
249        old_output = self.output
250        self.output = output
251        return old_output
252
253    def set_path(self, path):
254        """Set the path changed to before the child process is created."""
255        old_path = self.path
256        self.path = path
257        return old_path
258
259    def set_environ(self, environment):
260        """Set the environment passed to the child process when created."""
261        old_environment = self.environment
262        self.environment = environment
263        return old_environment
264
265class capture_execution(execute):
266    """Capture all output as a string and return it."""
267
268    class _output_snapper:
269        def __init__(self, log = None, dump = False):
270            self.output = ''
271            self.log = log
272            self.dump = dump
273
274        def handler(self, text):
275            if not self.dump:
276                if self.log is not None:
277                    self.log.output(text)
278                else:
279                    self.output += text
280
281        def get_and_clear(self):
282            text = self.output
283            self.output = ''
284            return text.strip()
285
286    def __init__(self, log = None, dump = False, error_prefix = '', verbose = False):
287        self.snapper = capture_execution._output_snapper(log = log, dump = dump)
288        execute.__init__(self, output = self.snapper.handler,
289                         error_prefix = error_prefix,
290                         verbose = verbose)
291
292    def open(self, command, capture = True, shell = False, cwd = None, env = None,
293             stdin = None, stdout = None, stderr = None):
294        if not capture:
295            raise error.general('output capture must be true; leave as default')
296        #self.snapper.get_and_clear()
297        exit_code, proc = execute.open(self, command, capture = True, shell = shell,
298                                       cwd = cwd, env = env,
299                                       stdin = stdin, stdout = stdout, stderr = stderr)
300        return (exit_code, proc, self.snapper.get_and_clear())
301
302    def set_output(self, output):
303        raise error.general('output capture cannot be overrided')
304
305if __name__ == "__main__":
306    def run_tests(e, commands, use_shell):
307        for c in commands['shell']:
308            e.shell(c)
309        for c in commands['spawn']:
310            e.spawn(c)
311        for c in commands['cmd']:
312            if type(c) is str:
313                e.command(c, shell = use_shell)
314            else:
315                e.command(c[0], c[1], shell = use_shell)
316        for c in commands['csubsts']:
317            e.command_subst(c[0], c[1], shell = use_shell)
318        ec, proc = e.command(commands['pipe'][0], commands['pipe'][1],
319                             capture = False, stdin = subprocess.PIPE)
320        if ec == 0:
321            print 'piping input into ' + commands['pipe'][0] + ': ' + \
322                  commands['pipe'][2]
323            proc.stdin.write(commands['pipe'][2])
324            proc.stdin.close()
325            e.capture(proc)
326            del proc
327
328    cmd_shell_test = 'if "%OS%" == "Windows_NT" (echo It is WinNT) else echo Is is not WinNT'
329    sh_shell_test = 'x="me"; if [ $x = "me" ]; then echo "It was me"; else "It was him"; fi'
330
331    commands = {}
332    commands['windows'] = {}
333    commands['unix'] = {}
334    commands['windows']['shell'] = ['cd', 'dir /w', '.\\xyz', cmd_shell_test]
335    commands['windows']['spawn'] = ['hostname', 'hostnameZZ', ['netstat', '/e']]
336    commands['windows']['cmd'] = [('ipconfig'), ('nslookup', 'www.python.org')]
337    commands['windows']['csubsts'] = [('netstat %0', ['-a']),
338                                      ('netstat %0 %1', ['-a', '-n'])]
339    commands['windows']['pipe'] = ('ftp', None, 'help\nquit')
340    commands['unix']['shell'] = ['pwd', 'ls -las', './xyz', sh_shell_test]
341    commands['unix']['spawn'] = ['ls', 'execute.pyc', ['ls', '-i']]
342    commands['unix']['cmd'] = [('date'), ('date', '-R'), ('date', ['-u', '+%d %D']),
343                               ('date', '-u "+%d %D %S"')]
344    commands['unix']['csubsts'] = [('date %0 "+%d %D %S"', ['-u']),
345                                   ('date %0 %1', ['-u', '+%d %D %S'])]
346    commands['unix']['pipe'] = ('grep', 'hello', 'hello world')
347
348    print arg_list('cmd a1 a2 "a3 is a string" a4')
349    print arg_list('cmd b1 b2 "b3 is a string a4')
350    print arg_subst(['nothing', 'xx-%0-yyy', '%1', '%2-something'],
351                    ['subst0', 'subst1', 'subst2'])
352
353    e = execute(error_prefix = 'ERR: ', verbose = True)
354    if sys.platform == "win32":
355        run_tests(e, commands['windows'], False)
356        if os.path.exists('c:\\msys\\1.0\\bin\\sh.exe'):
357            e.set_shell('c:\\msys\\1.0\\bin\\sh.exe --login -c')
358            commands['unix']['pipe'] = ('c:\\msys\\1.0\\bin\\grep',
359                                        'hello', 'hello world')
360            run_tests(e, commands['unix'], True)
361    else:
362        run_tests(e, commands['unix'], False)
363    del e
Note: See TracBrowser for help on using the repository browser.