Context Navigation

execute.py @ 4001a74

4.11

Last change on this file since 4001a74 was 4001a74, checked in by Chris Johns <chrisj@…>, on 03/02/16 at 09:54:06

Update rtems-tool to support Python 2 and 3.

Add solaris and netbsd.

Property mode set to 100755

File size: 21.7 KB

Rev	Line
[50fdf12]	1	#
	2	# RTEMS Tools Project (http://www.rtems.org/)
[4001a74]	3	# Copyright 2010-2016 Chris Johns (chrisj@rtems.org)
[50fdf12]	4	# All rights reserved.
	5	#
	6	# This file is part of the RTEMS Tools package in 'rtems-tools'.
	7	#
	8	# Redistribution and use in source and binary forms, with or without
	9	# modification, are permitted provided that the following conditions are met:
	10	#
	11	# 1. Redistributions of source code must retain the above copyright notice,
	12	# this list of conditions and the following disclaimer.
	13	#
	14	# 2. Redistributions in binary form must reproduce the above copyright notice,
	15	# this list of conditions and the following disclaimer in the documentation
	16	# and/or other materials provided with the distribution.
	17	#
	18	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
	19	# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	20	# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	21	# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
	22	# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
	23	# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
	24	# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
	25	# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
	26	# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
	27	# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	28	# POSSIBILITY OF SUCH DAMAGE.
	29	#
	30
	31	#
	32	# Execute commands or scripts.
	33	#
	34	# Note, the subprocess module is only in Python 2.4 or higher.
	35	#
	36
[4001a74]	37	from __future__ import print_function
	38
	39	import functools
[50fdf12]	40	import os
	41	import re
	42	import sys
	43	import subprocess
	44	import threading
	45	import time
	46
[4001a74]	47	#
	48	# Support to handle use in a package and as a unit test.
	49	# If there is a better way to let us know.
	50	#
	51	try:
	52	from . import error
	53	from . import log
	54	except (ValueError, SystemError):
	55	import error
	56	import log
[50fdf12]	57
	58	# Trace exceptions
	59	trace_threads = False
	60
	61	# Redefine the PIPE from subprocess
	62	PIPE = subprocess.PIPE
	63
	64	# Regular expression to find quotes.
	65	qstr = re.compile('[rR]?\'([^\\n\'\\\\]\|\\\\.)\'\|[rR]?"([^\\n"\\\\]\|\\\\.)"')
	66
	67	def check_type(command):
	68	"""Checks the type of command we have. The types are spawn and
	69	shell."""
	70	if command in ['spawn', 'shell']:
	71	return True
	72	return False
	73
	74	def arg_list(args):
	75	"""Turn a string of arguments into a list suitable for
	76	spawning a command. If the args are already a list return
	77	it."""
	78	if type(args) is list:
	79	return args
	80	argstr = args
	81	args = []
	82	while len(argstr):
	83	qs = qstr.search(argstr)
	84	if not qs:
	85	args.extend(argstr.split())
	86	argstr= ''
	87	else:
	88	# We have a quoted string. Get the string before
	89	# the quoted string and splt on white space then
	90	# add the quoted string as an option then remove
	91	# the first + quoted string and try again
	92	front = argstr[:qs.start()]
	93	args.extend(front.split())
	94	args.append(argstr[qs.start() + 1:qs.end() - 1])
	95	argstr = argstr[qs.end():]
	96	return args
	97
	98	def arg_subst(command, substs):
	99	"""Substitute the %[0-9] in the command with the subst values."""
	100	args = arg_list(command)
	101	if substs:
	102	for a in range(0, len(args)):
	103	for r in range(0, len(substs)):
	104	args[a] = re.compile(('%%%d' % (r))).sub(substs[r], args[a])
	105	return args
	106
	107	def arg_subst_str(command, subst):
	108	cmd = arg_subst(command, subst)
	109	def add(x, y): return x + ' ' + str(y)
[4001a74]	110	return functools.reduce(add, cmd, '')
[50fdf12]	111
	112	class execute(object):
	113	"""Execute commands or scripts. The 'output' is a funtion that handles the
	114	output from the process. The 'input' is a function that blocks and returns
	115	data to be written to stdin"""
	116	def __init__(self, output = None, input = None, cleanup = None,
	117	error_prefix = '', verbose = False):
	118	self.lock = threading.Lock()
	119	self.output = output
	120	self.input = input
	121	self.cleanup = cleanup
	122	self.error_prefix = error_prefix
	123	self.verbose = verbose
	124	self.shell_exe = None
	125	self.shell_commands = False
	126	self.path = None
	127	self.environment = None
	128	self.outputting = False
	129	self.timing_out = False
[c04a849]	130	self.proc = None
[50fdf12]	131
[4001a74]	132	def capture(self, proc, command = 'pipe', timeout = None):
[50fdf12]	133	"""Create 3 threads to read stdout and stderr and send to the output handler
	134	and call an input handler is provided. Based on the 'communicate' code
	135	in the subprocess module."""
	136	def _writethread(exe, fh, input):
	137	"""Call the input handler and write it to the stdin. The input handler should
	138	block and return None or False if this thread is to exit and True if this
	139	is a timeout check."""
	140	if trace_threads:
[4001a74]	141	print('executte:_writethread: start')
[50fdf12]	142	try:
	143	while True:
[4001a74]	144	lines = eval(input())
[50fdf12]	145	if type(lines) == str:
	146	try:
[4001a74]	147	fh.write(bytes(lines, sys.stdin.encoding))
[50fdf12]	148	except:
	149	break
	150	if lines == None or \
	151	lines == False or \
	152	(lines == True and fh.closed):
	153	break
	154	except:
	155	if trace_threads:
[4001a74]	156	print('executte:_writethread: exception')
[50fdf12]	157	pass
	158	try:
	159	fh.close()
	160	except:
	161	pass
	162	if trace_threads:
[4001a74]	163	print('executte:_writethread: finished')
[50fdf12]	164
	165	def _readthread(exe, fh, out, prefix = ''):
	166	"""Read from a file handle and write to the output handler
	167	until the file closes."""
	168	def _output_line(line, exe, prefix, out, count):
[4001a74]	169	#exe.lock.acquire()
[50fdf12]	170	#exe.outputting = True
[4001a74]	171	#exe.lock.release()
[50fdf12]	172	if out:
	173	out(prefix + line)
	174	else:
	175	log.output(prefix + line)
	176	if count > 10:
	177	log.flush()
	178
	179	if trace_threads:
[4001a74]	180	print('executte:_readthread: start')
[50fdf12]	181	count = 0
	182	line = ''
	183	try:
	184	while True:
	185	data = fh.read(1)
	186	if len(data) == 0:
	187	break
[4001a74]	188	if type(data) == bytes:
	189	data = data.decode(sys.stdout.encoding)
[50fdf12]	190	for c in data:
	191	line += c
	192	if c == '\n':
	193	count += 1
	194	_output_line(line, exe, prefix, out, count)
	195	if count > 10:
	196	count = 0
	197	line = ''
	198	except:
	199	raise
	200	if trace_threads:
[4001a74]	201	print('executte:_readthread: exception')
[50fdf12]	202	pass
	203	try:
	204	fh.close()
	205	except:
	206	pass
	207	if len(line):
	208	_output_line(line, exe, prefix, out, 100)
	209	if trace_threads:
[4001a74]	210	print('executte:_readthread: finished')
[50fdf12]	211
	212	def _timerthread(exe, interval, function):
	213	"""Timer thread is used to timeout a process if no output is
	214	produced for the timeout interval."""
	215	count = interval
	216	while exe.timing_out:
	217	time.sleep(1)
	218	if count > 0:
	219	count -= 1
	220	exe.lock.acquire()
	221	if exe.outputting:
	222	count = interval
	223	exe.outputting = False
	224	exe.lock.release()
	225	if count == 0:
	226	try:
	227	proc.kill()
	228	except:
	229	pass
	230	else:
	231	function()
	232	break
	233
	234	name = os.path.basename(command[0])
	235
	236	stdin_thread = None
	237	stdout_thread = None
	238	stderr_thread = None
	239	timeout_thread = None
	240
	241	if proc.stdout:
	242	stdout_thread = threading.Thread(target = _readthread,
	243	name = '_stdout[%s]' % (name),
	244	args = (self,
	245	proc.stdout,
	246	self.output,
	247	''))
	248	stdout_thread.daemon = True
	249	stdout_thread.start()
	250	if proc.stderr:
	251	stderr_thread = threading.Thread(target = _readthread,
	252	name = '_stderr[%s]' % (name),
	253	args = (self,
	254	proc.stderr,
	255	self.output,
	256	self.error_prefix))
	257	stderr_thread.daemon = True
	258	stderr_thread.start()
	259	if self.input and proc.stdin:
	260	stdin_thread = threading.Thread(target = _writethread,
	261	name = '_stdin[%s]' % (name),
	262	args = (self,
	263	proc.stdin,
	264	self.input))
	265	stdin_thread.daemon = True
	266	stdin_thread.start()
	267	if timeout:
	268	self.timing_out = True
	269	timeout_thread = threading.Thread(target = _timerthread,
	270	name = '_timeout[%s]' % (name),
	271	args = (self,
	272	timeout[0],
	273	timeout[1]))
	274	timeout_thread.daemon = True
	275	timeout_thread.start()
	276	try:
[c04a849]	277	self.lock.acquire()
	278	try:
	279	self.proc = proc
	280	except:
	281	raise
	282	finally:
	283	self.lock.release()
[50fdf12]	284	exitcode = proc.wait()
	285	except:
	286	proc.kill()
	287	raise
	288	finally:
[c04a849]	289	self.lock.acquire()
	290	try:
	291	self.proc = None
	292	except:
	293	raise
	294	finally:
	295	self.lock.release()
[50fdf12]	296	if self.cleanup:
	297	self.cleanup(proc)
	298	if timeout_thread:
	299	self.timing_out = False
	300	timeout_thread.join(10)
	301	if stdin_thread:
	302	stdin_thread.join(2)
	303	if stdout_thread:
	304	stdout_thread.join(2)
	305	if stderr_thread:
	306	stderr_thread.join(2)
	307	return exitcode
	308
	309	def open(self, command, capture = True, shell = False,
	310	cwd = None, env = None,
	311	stdin = None, stdout = None, stderr = None,
	312	timeout = None):
	313	"""Open a command with arguments. Provide the arguments as a list or
	314	a string."""
	315	if self.verbose:
	316	s = command
	317	if type(command) is list:
	318	def add(x, y): return x + ' ' + str(y)
[4001a74]	319	s = functools.reduce(add, command, '')[1:]
[50fdf12]	320	what = 'spawn'
	321	if shell:
	322	what = 'shell'
	323	log.output(what + ': ' + s)
[4001a74]	324	if self.output is None:
	325	raise error.general('capture needs an output handler')
[50fdf12]	326	if shell and self.shell_exe:
	327	command = arg_list(command)
	328	command[:0] = self.shell_exe
	329	if not stdin and self.input:
	330	stdin = subprocess.PIPE
	331	if not stdout:
	332	stdout = subprocess.PIPE
	333	if not stderr:
	334	stderr = subprocess.PIPE
	335	proc = None
	336	if cwd is None:
	337	cwd = self.path
	338	if env is None:
	339	env = self.environment
	340	try:
	341	# Work around a problem on Windows with commands that
	342	# have a '.' and no extension. Windows needs the full
	343	# command name.
	344	if sys.platform == "win32" and type(command) is list:
	345	if command[0].find('.') >= 0:
	346	r, e = os.path.splitext(command[0])
	347	if e not in ['.exe', '.com', '.bat']:
	348	command[0] = command[0] + '.exe'
	349	log.trace('exe: %s' % (command))
	350	proc = subprocess.Popen(command, shell = shell,
	351	cwd = cwd, env = env,
	352	stdin = stdin, stdout = stdout,
	353	stderr = stderr)
	354	if not capture:
	355	return (0, proc)
[4001a74]	356	exit_code = self.capture(proc, command, timeout)
[50fdf12]	357	if self.verbose:
	358	log.output('exit: ' + str(exit_code))
[4001a74]	359	except OSError as ose:
[50fdf12]	360	exit_code = ose.errno
	361	if self.verbose:
	362	log.output('exit: ' + str(ose))
	363	return (exit_code, proc)
	364
	365	def spawn(self, command, capture = True, cwd = None, env = None,
	366	stdin = None, stdout = None, stderr = None,
	367	timeout = None):
	368	"""Spawn a command with arguments. Provide the arguments as a list or
	369	a string."""
	370	return self.open(command, capture, False, cwd, env,
	371	stdin, stdout, stderr, timeout)
	372
	373	def shell(self, command, capture = True, cwd = None, env = None,
	374	stdin = None, stdout = None, stderr = None,
	375	timeout = None):
	376	"""Execute a command within a shell context. The command can contain
	377	argumments. The shell is specific to the operating system. For example
	378	it is cmd.exe on Windows XP."""
	379	return self.open(command, capture, True, cwd, env,
	380	stdin, stdout, stderr, timeout)
	381
	382	def command(self, command, args = None, capture = True, shell = False,
	383	cwd = None, env = None,
	384	stdin = None, stdout = None, stderr = None,
	385	timeout = None):
	386	"""Run the command with the args. The args can be a list
	387	or a string."""
	388	if args and not type(args) is list:
	389	args = arg_list(args)
	390	cmd = [command]
	391	if args:
	392	cmd.extend(args)
	393	return self.open(cmd, capture = capture, shell = shell,
	394	cwd = cwd, env = env,
	395	stdin = stdin, stdout = stdout, stderr = stderr,
	396	timeout = timeout)
	397
	398	def command_subst(self, command, substs, capture = True, shell = False,
	399	cwd = None, env = None,
	400	stdin = None, stdout = None, stderr = None,
	401	timeout = None):
	402	"""Run the command from the config data with the
	403	option format string subsituted with the subst variables."""
	404	args = arg_subst(command, substs)
	405	return self.command(args[0], args[1:], capture = capture,
	406	shell = shell or self.shell_commands,
	407	cwd = cwd, env = env,
	408	stdin = stdin, stdout = stdout, stderr = stderr,
[4001a74]	409	timeout = timeout)
[50fdf12]	410
	411	def set_shell(self, execute):
	412	"""Set the shell to execute when issuing a shell command."""
	413	args = arg_list(execute)
	414	if len(args) == 0 or not os.path.isfile(args[0]):
	415	raise error.general('could find shell: ' + execute)
	416	self.shell_exe = args
	417
	418	def command_use_shell(self):
	419	"""Force all commands to use a shell. This can be used with set_shell
	420	to allow Unix commands be executed on Windows with a Unix shell such
	421	as Cygwin or MSYS. This may cause piping to fail."""
	422	self.shell_commands = True
	423
	424	def set_output(self, output):
	425	"""Set the output handler. The stdout of the last process in a pipe
	426	line is passed to this handler."""
	427	old_output = self.output
	428	self.output = output
	429	return old_output
	430
	431	def set_path(self, path):
	432	"""Set the path changed to before the child process is created."""
	433	old_path = self.path
	434	self.path = path
	435	return old_path
	436
	437	def set_environ(self, environment):
	438	"""Set the environment passed to the child process when created."""
	439	old_environment = self.environment
	440	self.environment = environment
	441	return old_environment
	442
[c04a849]	443	def kill(self):
	444	self.lock.acquire()
	445	try:
	446	if self.proc is not None:
	447	self.proc.kill()
	448	except:
	449	raise
	450	finally:
	451	self.lock.release()
	452
	453	def terminate(self):
	454	self.lock.acquire()
	455	try:
	456	if self.proc is not None:
	457	self.proc.terminate()
	458	except:
	459	raise
	460	finally:
	461	self.lock.release()
	462
	463	def send_signal(self, signal):
	464	self.lock.acquire()
	465	try:
	466	if self.proc is not None:
[4001a74]	467	print("sending sig")
[c04a849]	468	self.proc.send_signal(signal)
	469	except:
	470	raise
	471	finally:
	472	self.lock.release()
	473
[50fdf12]	474	class capture_execution(execute):
	475	"""Capture all output as a string and return it."""
	476
	477	class _output_snapper:
	478	def __init__(self, log = None, dump = False):
	479	self.output = ''
	480	self.log = log
	481	self.dump = dump
	482
	483	def handler(self, text):
	484	if not self.dump:
	485	if self.log is not None:
	486	self.log.output(text)
	487	else:
	488	self.output += text
	489
	490	def get_and_clear(self):
	491	text = self.output
	492	self.output = ''
	493	return text.strip()
	494
	495	def __init__(self, log = None, dump = False, error_prefix = '', verbose = False):
	496	self.snapper = capture_execution._output_snapper(log = log, dump = dump)
	497	execute.__init__(self, output = self.snapper.handler,
	498	error_prefix = error_prefix,
	499	verbose = verbose)
	500
	501	def open(self, command, capture = True, shell = False, cwd = None, env = None,
	502	stdin = None, stdout = None, stderr = None, timeout = None):
	503	if not capture:
	504	raise error.general('output capture must be true; leave as default')
	505	#self.snapper.get_and_clear()
	506	exit_code, proc = execute.open(self, command, capture = True, shell = shell,
	507	cwd = cwd, env = env,
	508	stdin = stdin, stdout = stdout, stderr = stderr,
	509	timeout = timeout)
	510	return (exit_code, proc, self.snapper.get_and_clear())
	511
	512	def set_output(self, output):
	513	raise error.general('output capture cannot be overrided')
	514
	515	if __name__ == "__main__":
[4001a74]	516
[50fdf12]	517	def run_tests(e, commands, use_shell):
	518	for c in commands['shell']:
	519	e.shell(c)
	520	for c in commands['spawn']:
	521	e.spawn(c)
	522	for c in commands['cmd']:
	523	if type(c) is str:
	524	e.command(c, shell = use_shell)
	525	else:
	526	e.command(c[0], c[1], shell = use_shell)
	527	for c in commands['csubsts']:
	528	e.command_subst(c[0], c[1], shell = use_shell)
	529	ec, proc = e.command(commands['pipe'][0], commands['pipe'][1],
	530	capture = False, stdin = subprocess.PIPE)
	531	if ec == 0:
[4001a74]	532	print('piping input into ' + commands['pipe'][0] + ': ' + \
	533	commands['pipe'][2])
	534	try:
	535	out = bytes(commands['pipe'][2], sys.stdin.encoding)
	536	except:
	537	out = bytes(commands['pipe'][2])
	538	proc.stdin.write(out)
[50fdf12]	539	proc.stdin.close()
	540	e.capture(proc)
	541	del proc
	542
[4001a74]	543	def capture_output(text):
	544	print(text, end = '')
	545
[50fdf12]	546	cmd_shell_test = 'if "%OS%" == "Windows_NT" (echo It is WinNT) else echo Is is not WinNT'
	547	sh_shell_test = 'x="me"; if [ $x = "me" ]; then echo "It was me"; else "It was him"; fi'
	548
	549	commands = {}
	550	commands['windows'] = {}
	551	commands['unix'] = {}
	552	commands['windows']['shell'] = ['cd', 'dir /w', '.\\xyz', cmd_shell_test]
	553	commands['windows']['spawn'] = ['hostname', 'hostnameZZ', ['netstat', '/e']]
	554	commands['windows']['cmd'] = [('ipconfig'), ('nslookup', 'www.python.org')]
	555	commands['windows']['csubsts'] = [('netstat %0', ['-a']),
	556	('netstat %0 %1', ['-a', '-n'])]
	557	commands['windows']['pipe'] = ('ftp', None, 'help\nquit')
	558	commands['unix']['shell'] = ['pwd', 'ls -las', './xyz', sh_shell_test]
	559	commands['unix']['spawn'] = ['ls', 'execute.pyc', ['ls', '-i']]
	560	commands['unix']['cmd'] = [('date'), ('date', '-R'), ('date', ['-u', '+%d %D']),
	561	('date', '-u "+%d %D %S"')]
	562	commands['unix']['csubsts'] = [('date %0 "+%d %D %S"', ['-u']),
	563	('date %0 %1', ['-u', '+%d %D %S'])]
	564	commands['unix']['pipe'] = ('grep', 'hello', 'hello world')
	565
[4001a74]	566	print(arg_list('cmd a1 a2 "a3 is a string" a4'))
	567	print(arg_list('cmd b1 b2 "b3 is a string a4'))
	568	print(arg_subst(['nothing', 'xx-%0-yyy', '%1', '%2-something'],
	569	['subst0', 'subst1', 'subst2']))
[50fdf12]	570
[4001a74]	571	e = execute(error_prefix = 'ERR: ', output = capture_output, verbose = True)
[50fdf12]	572	if sys.platform == "win32":
	573	run_tests(e, commands['windows'], False)
	574	if os.path.exists('c:\\msys\\1.0\\bin\\sh.exe'):
	575	e.set_shell('c:\\msys\\1.0\\bin\\sh.exe --login -c')
	576	commands['unix']['pipe'] = ('c:\\msys\\1.0\\bin\\grep',
	577	'hello', 'hello world')
	578	run_tests(e, commands['unix'], True)
	579	else:
	580	run_tests(e, commands['unix'], False)
	581	del e

Note: See TracBrowser for help on using the repository browser.

Context Navigation

source: rtems-tools/rtemstoolkit/execute.py @ 4001a74

Download in other formats: