Changeset c2d2338 in rtems-source-builder
- Timestamp:
- 12/18/18 04:08:43 (4 years ago)
- Branches:
- 5, master
- Children:
- 257c926
- Parents:
- 2fac55d
- git-author:
- Chris Johns <chrisj@…> (12/18/18 04:08:43)
- git-committer:
- Chris Johns <chrisj@…> (12/24/18 23:15:55)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
source-builder/sb/execute.py
r2fac55d rc2d2338 1 1 # 2 2 # RTEMS Tools Project (http://www.rtems.org/) 3 # Copyright 2010-201 6Chris Johns (chrisj@rtems.org)3 # Copyright 2010-2017 Chris Johns (chrisj@rtems.org) 4 4 # All rights reserved. 5 5 # … … 9 9 # purpose with or without fee is hereby granted, provided that the above 10 10 # copyright notice and this permission notice appear in all copies. 11 #11 # 12 12 # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 13 13 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF … … 17 17 # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 18 18 # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 19 # 19 20 20 21 # … … 27 28 28 29 import functools 30 import io 29 31 import os 30 32 import re … … 32 34 import subprocess 33 35 import threading 36 import time 37 import traceback 34 38 35 39 import error 36 40 import log 41 42 # Trace exceptions 43 trace_threads = False 37 44 38 45 # Redefine the PIPE from subprocess … … 87 94 return functools.reduce(add, cmd, '') 88 95 89 class execute: 90 """Execute commands or scripts. The 'output' is a funtion 91 that handles the output from the process.""" 92 def __init__(self, output = None, error_prefix = '', verbose = False): 96 class execute(object): 97 """Execute commands or scripts. The 'output' is a funtion that handles the 98 output from the process. The 'input' is a function that blocks and returns 99 data to be written to stdin""" 100 def __init__(self, output = None, input = None, cleanup = None, 101 error_prefix = '', verbose = False): 102 self.lock = threading.Lock() 93 103 self.output = output 104 self.input = input 105 self.cleanup = cleanup 94 106 self.error_prefix = error_prefix 95 107 self.verbose = verbose … … 98 110 self.path = None 99 111 self.environment = None 100 101 def capture(self, proc, timeout = None): 102 """Create 2 threads to read stdout and stderr and send to the 103 output handler. Based on the 'communicate' code in the subprocess 104 module.""" 105 def _readthread(fh, out, prefix = ''): 112 self.outputting = False 113 self.timing_out = False 114 self.proc = None 115 116 def capture(self, proc, command = 'pipe', timeout = None): 117 """Create 3 threads to read stdout and stderr and send to the output handler 118 and call an input handler is provided. Based on the 'communicate' code 119 in the subprocess module.""" 120 def _writethread(exe, fh, input): 121 """Call the input handler and write it to the stdin. The input handler should 122 block and return None or False if this thread is to exit and True if this 123 is a timeout check.""" 124 if trace_threads: 125 print('execute:_writethread: start') 126 encoding = True 127 try: 128 tmp = bytes('temp', sys.stdin.encoding) 129 except: 130 encoding = False 131 try: 132 while True: 133 if trace_threads: 134 print('execute:_writethread: call input', input) 135 lines = input() 136 if type(lines) == str or type(lines) == bytes: 137 try: 138 if encoding: 139 lines = bytes(lines, sys.stdin.encoding) 140 fh.write(lines) 141 fh.flush() 142 except: 143 break 144 if lines == None or \ 145 lines == False or \ 146 (lines == True and fh.closed): 147 break 148 except: 149 if trace_threads: 150 print('execute:_writethread: exception') 151 print(traceback.format_exc()) 152 pass 153 try: 154 fh.close() 155 except: 156 pass 157 if trace_threads: 158 print('execute:_writethread: finished') 159 160 def _readthread(exe, fh, out, prefix = ''): 106 161 """Read from a file handle and write to the output handler 107 162 until the file closes.""" 108 count = 0 109 while True: 110 line = fh.readline() 111 # str and bytes are the same type in Python2 112 if type(line) is not str and type(line) is bytes: 113 line = line.decode(sys.stdout.encoding) 114 count += 1 115 if len(line) == 0: 116 break 163 def _output_line(line, exe, prefix, out, count): 164 #exe.lock.acquire() 165 #exe.outputting = True 166 #exe.lock.release() 117 167 if out: 118 168 out(prefix + line) … … 121 171 if count > 10: 122 172 log.flush() 123 count = 0 124 125 def _timerthread(proc, timer): 126 """Timer thread calls the timer handler if one 127 is present once a second. The user provides a handler 128 and returns False to kill the process or True continue.""" 129 while True: 173 174 if trace_threads: 175 print('execute:_readthread: start') 176 count = 0 177 line = '' 178 try: 179 while True: 180 # 181 # The io module file handling return up to the size passed 182 # in to the read call. The io handle has the default 183 # buffering size. On any error assume the handle has gone 184 # and the process is shutting down. 185 # 186 try: 187 data = fh.read1(4096) 188 except: 189 data = '' 190 if len(data) == 0: 191 if len(line) > 0: 192 _output_line(line + '\n', exe, prefix, out, count) 193 break 194 # str and bytes are the same type in Python2 195 if type(data) is not str and type(data) is bytes: 196 data = data.decode(sys.stdout.encoding) 197 last_ch = data[-1] 198 sd = (line + data).split('\n') 199 if last_ch != '\n': 200 line = sd[-1] 201 else: 202 line = '' 203 sd = sd[:-1] 204 if len(sd) > 0: 205 for l in sd: 206 _output_line(l + '\n', exe, prefix, out, count) 207 count += 1 208 if count > 10: 209 count -= 10 210 except: 211 raise 212 if trace_threads: 213 print('execute:_readthread: exception') 214 print(traceback.format_exc()) 215 pass 216 try: 217 fh.close() 218 except: 219 pass 220 if len(line): 221 _output_line(line, exe, prefix, out, 100) 222 if trace_threads: 223 print('execute:_readthread: finished') 224 225 def _timerthread(exe, interval, function): 226 """Timer thread is used to timeout a process if no output is 227 produced for the timeout interval.""" 228 count = interval 229 while exe.timing_out: 130 230 time.sleep(1) 131 if not timer(proc): 132 proc.stdout.close() 133 proc.stderr.close() 231 if count > 0: 232 count -= 1 233 exe.lock.acquire() 234 if exe.outputting: 235 count = interval 236 exe.outputting = False 237 exe.lock.release() 238 if count == 0: 239 try: 240 proc.kill() 241 except: 242 pass 243 else: 244 function() 245 break 246 247 name = os.path.basename(command[0]) 248 249 stdin_thread = None 250 stdout_thread = None 251 stderr_thread = None 252 timeout_thread = None 134 253 135 254 if proc.stdout: 136 255 stdout_thread = threading.Thread(target = _readthread, 137 args = (proc.stdout, 256 name = '_stdout[%s]' % (name), 257 args = (self, 258 io.open(proc.stdout.fileno(), 259 mode = 'rb', 260 closefd = False), 138 261 self.output, 139 262 '')) 140 stdout_thread. setDaemon(True)263 stdout_thread.daemon = True 141 264 stdout_thread.start() 142 265 if proc.stderr: 143 266 stderr_thread = threading.Thread(target = _readthread, 144 args = (proc.stderr, 267 name = '_stderr[%s]' % (name), 268 args = (self, 269 io.open(proc.stderr.fileno(), 270 mode = 'rb', 271 closefd = False), 145 272 self.output, 146 273 self.error_prefix)) 147 stderr_thread. setDaemon(True)274 stderr_thread.daemon = True 148 275 stderr_thread.start() 149 if proc.stdout: 150 stdout_thread.join() 151 if proc.stderr: 152 stderr_thread.join() 153 return proc.wait() 276 if self.input and proc.stdin: 277 stdin_thread = threading.Thread(target = _writethread, 278 name = '_stdin[%s]' % (name), 279 args = (self, 280 proc.stdin, 281 self.input)) 282 stdin_thread.daemon = True 283 stdin_thread.start() 284 if timeout: 285 self.timing_out = True 286 timeout_thread = threading.Thread(target = _timerthread, 287 name = '_timeout[%s]' % (name), 288 args = (self, 289 timeout[0], 290 timeout[1])) 291 timeout_thread.daemon = True 292 timeout_thread.start() 293 try: 294 self.lock.acquire() 295 try: 296 self.proc = proc 297 except: 298 raise 299 finally: 300 self.lock.release() 301 exitcode = proc.wait() 302 except: 303 proc.kill() 304 raise 305 finally: 306 self.lock.acquire() 307 try: 308 self.proc = None 309 except: 310 raise 311 finally: 312 self.lock.release() 313 if self.cleanup: 314 self.cleanup(proc) 315 if timeout_thread: 316 self.timing_out = False 317 timeout_thread.join(10) 318 if stdin_thread: 319 stdin_thread.join(2) 320 if stdout_thread: 321 stdout_thread.join(2) 322 if stderr_thread: 323 stderr_thread.join(2) 324 return exitcode 154 325 155 326 def open(self, command, capture = True, shell = False, 156 327 cwd = None, env = None, 157 stdin = None, stdout = None, stderr = None): 328 stdin = None, stdout = None, stderr = None, 329 timeout = None): 158 330 """Open a command with arguments. Provide the arguments as a list or 159 331 a string.""" … … 167 339 what = 'shell' 168 340 log.output(what + ': ' + s) 341 if self.output is None: 342 raise error.general('capture needs an output handler') 169 343 if shell and self.shell_exe: 170 344 command = arg_list(command) 171 345 command[:0] = self.shell_exe 346 if not stdin and self.input: 347 stdin = subprocess.PIPE 172 348 if not stdout: 173 349 stdout = subprocess.PIPE … … 192 368 cwd = cwd, env = env, 193 369 stdin = stdin, stdout = stdout, 194 stderr = stderr) 370 stderr = stderr, 371 close_fds = False) 195 372 if not capture: 196 373 return (0, proc) 197 exit_code = self.capture(proc) 374 if self.output is None: 375 raise error.general('capture needs an output handler') 376 exit_code = self.capture(proc, command, timeout) 198 377 if self.verbose: 199 378 log.output('exit: ' + str(exit_code)) … … 205 384 206 385 def spawn(self, command, capture = True, cwd = None, env = None, 207 stdin = None, stdout = None, stderr = None): 386 stdin = None, stdout = None, stderr = None, 387 timeout = None): 208 388 """Spawn a command with arguments. Provide the arguments as a list or 209 389 a string.""" 210 390 return self.open(command, capture, False, cwd, env, 211 stdin, stdout, stderr )391 stdin, stdout, stderr, timeout) 212 392 213 393 def shell(self, command, capture = True, cwd = None, env = None, 214 stdin = None, stdout = None, stderr = None): 394 stdin = None, stdout = None, stderr = None, 395 timeout = None): 215 396 """Execute a command within a shell context. The command can contain 216 397 argumments. The shell is specific to the operating system. For example 217 398 it is cmd.exe on Windows XP.""" 218 399 return self.open(command, capture, True, cwd, env, 219 stdin, stdout, stderr )400 stdin, stdout, stderr, timeout) 220 401 221 402 def command(self, command, args = None, capture = True, shell = False, 222 403 cwd = None, env = None, 223 stdin = None, stdout = None, stderr = None): 404 stdin = None, stdout = None, stderr = None, 405 timeout = None): 224 406 """Run the command with the args. The args can be a list 225 407 or a string.""" … … 231 413 return self.open(cmd, capture = capture, shell = shell, 232 414 cwd = cwd, env = env, 233 stdin = stdin, stdout = stdout, stderr = stderr) 415 stdin = stdin, stdout = stdout, stderr = stderr, 416 timeout = timeout) 234 417 235 418 def command_subst(self, command, substs, capture = True, shell = False, 236 419 cwd = None, env = None, 237 stdin = None, stdout = None, stderr = None): 420 stdin = None, stdout = None, stderr = None, 421 timeout = None): 238 422 """Run the command from the config data with the 239 423 option format string subsituted with the subst variables.""" … … 242 426 shell = shell or self.shell_commands, 243 427 cwd = cwd, env = env, 244 stdin = stdin, stdout = stdout, stderr = stderr) 428 stdin = stdin, stdout = stdout, stderr = stderr, 429 timeout = timeout) 245 430 246 431 def set_shell(self, execute): … … 276 461 return old_environment 277 462 463 def kill(self): 464 self.lock.acquire() 465 try: 466 if self.proc is not None: 467 self.proc.kill() 468 except: 469 raise 470 finally: 471 self.lock.release() 472 473 def terminate(self): 474 self.lock.acquire() 475 try: 476 if self.proc is not None: 477 self.proc.terminate() 478 except: 479 raise 480 finally: 481 self.lock.release() 482 483 def send_signal(self, signal): 484 self.lock.acquire() 485 try: 486 if self.proc is not None: 487 print("sending sig") 488 self.proc.send_signal(signal) 489 except: 490 raise 491 finally: 492 self.lock.release() 493 278 494 class capture_execution(execute): 279 495 """Capture all output as a string and return it.""" … … 304 520 305 521 def open(self, command, capture = True, shell = False, cwd = None, env = None, 306 stdin = None, stdout = None, stderr = None ):522 stdin = None, stdout = None, stderr = None, timeout = None): 307 523 if not capture: 308 524 raise error.general('output capture must be true; leave as default') … … 310 526 exit_code, proc = execute.open(self, command, capture = True, shell = shell, 311 527 cwd = cwd, env = env, 312 stdin = stdin, stdout = stdout, stderr = stderr) 528 stdin = stdin, stdout = stdout, stderr = stderr, 529 timeout = timeout) 313 530 return (exit_code, proc, self.snapper.get_and_clear()) 314 531 … … 334 551 print('piping input into ' + commands['pipe'][0] + ': ' + \ 335 552 commands['pipe'][2]) 336 proc.stdin.write(bytes(commands['pipe'][2], sys.stdin.encoding)) 553 try: 554 out = bytes(commands['pipe'][2], sys.stdin.encoding) 555 except: 556 out = commands['pipe'][2] 557 proc.stdin.write(out) 337 558 proc.stdin.close() 338 559 e.capture(proc) 339 560 del proc 561 562 def capture_output(text): 563 print(text, end = '') 340 564 341 565 cmd_shell_test = 'if "%OS%" == "Windows_NT" (echo It is WinNT) else echo Is is not WinNT' … … 364 588 ['subst0', 'subst1', 'subst2'])) 365 589 366 e = execute(error_prefix = 'ERR: ', verbose = True)590 e = execute(error_prefix = 'ERR: ', output = capture_output, verbose = True) 367 591 if sys.platform == "win32": 368 592 run_tests(e, commands['windows'], False)
Note: See TracChangeset
for help on using the changeset viewer.