738 lines
28 KiB
Python
738 lines
28 KiB
Python
"""
|
|
Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
See https://llvm.org/LICENSE.txt for license information.
|
|
SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
Provides classes used by the test results reporting infrastructure
|
|
within the LLDB test suite.
|
|
|
|
|
|
This module provides process-management support for the LLDB test
|
|
running infrastructure.
|
|
"""
|
|
|
|
# System imports
|
|
import os
|
|
import re
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
|
|
|
|
class CommunicatorThread(threading.Thread):
|
|
"""Provides a thread class that communicates with a subprocess."""
|
|
|
|
def __init__(self, process, event, output_file):
|
|
super(CommunicatorThread, self).__init__()
|
|
# Don't let this thread prevent shutdown.
|
|
self.daemon = True
|
|
self.process = process
|
|
self.pid = process.pid
|
|
self.event = event
|
|
self.output_file = output_file
|
|
self.output = None
|
|
|
|
def run(self):
|
|
try:
|
|
# Communicate with the child process.
|
|
# This will not complete until the child process terminates.
|
|
self.output = self.process.communicate()
|
|
except Exception as exception: # pylint: disable=broad-except
|
|
if self.output_file:
|
|
self.output_file.write(
|
|
"exception while using communicate() for pid: {}\n".format(
|
|
exception))
|
|
finally:
|
|
# Signal that the thread's run is complete.
|
|
self.event.set()
|
|
|
|
|
|
# Provides a regular expression for matching gtimeout-based durations.
|
|
TIMEOUT_REGEX = re.compile(r"(^\d+)([smhd])?$")
|
|
|
|
|
|
def timeout_to_seconds(timeout):
|
|
"""Converts timeout/gtimeout timeout values into seconds.
|
|
|
|
@param timeout a timeout in the form of xm representing x minutes.
|
|
|
|
@return None if timeout is None, or the number of seconds as a float
|
|
if a valid timeout format was specified.
|
|
"""
|
|
if timeout is None:
|
|
return None
|
|
else:
|
|
match = TIMEOUT_REGEX.match(timeout)
|
|
if match:
|
|
value = float(match.group(1))
|
|
units = match.group(2)
|
|
if units is None:
|
|
# default is seconds. No conversion necessary.
|
|
return value
|
|
elif units == 's':
|
|
# Seconds. No conversion necessary.
|
|
return value
|
|
elif units == 'm':
|
|
# Value is in minutes.
|
|
return 60.0 * value
|
|
elif units == 'h':
|
|
# Value is in hours.
|
|
return (60.0 * 60.0) * value
|
|
elif units == 'd':
|
|
# Value is in days.
|
|
return 24 * (60.0 * 60.0) * value
|
|
else:
|
|
raise Exception("unexpected units value '{}'".format(units))
|
|
else:
|
|
raise Exception("could not parse TIMEOUT spec '{}'".format(
|
|
timeout))
|
|
|
|
|
|
class ProcessHelper(object):
|
|
"""Provides an interface for accessing process-related functionality.
|
|
|
|
This class provides a factory method that gives the caller a
|
|
platform-specific implementation instance of the class.
|
|
|
|
Clients of the class should stick to the methods provided in this
|
|
base class.
|
|
|
|
\see ProcessHelper.process_helper()
|
|
"""
|
|
|
|
def __init__(self):
|
|
super(ProcessHelper, self).__init__()
|
|
|
|
@classmethod
|
|
def process_helper(cls):
|
|
"""Returns a platform-specific ProcessHelper instance.
|
|
@return a ProcessHelper instance that does the right thing for
|
|
the current platform.
|
|
"""
|
|
|
|
# If you add a new platform, create an instance here and
|
|
# return it.
|
|
if os.name == "nt":
|
|
return WindowsProcessHelper()
|
|
else:
|
|
# For all POSIX-like systems.
|
|
return UnixProcessHelper()
|
|
|
|
def create_piped_process(self, command, new_process_group=True):
|
|
# pylint: disable=no-self-use,unused-argument
|
|
# As expected. We want derived classes to implement this.
|
|
"""Creates a subprocess.Popen-based class with I/O piped to the parent.
|
|
|
|
@param command the command line list as would be passed to
|
|
subprocess.Popen(). Use the list form rather than the string form.
|
|
|
|
@param new_process_group indicates if the caller wants the
|
|
process to be created in its own process group. Each OS handles
|
|
this concept differently. It provides a level of isolation and
|
|
can simplify or enable terminating the process tree properly.
|
|
|
|
@return a subprocess.Popen-like object.
|
|
"""
|
|
raise Exception("derived class must implement")
|
|
|
|
def supports_soft_terminate(self):
|
|
# pylint: disable=no-self-use
|
|
# As expected. We want derived classes to implement this.
|
|
"""Indicates if the platform supports soft termination.
|
|
|
|
Soft termination is the concept of a terminate mechanism that
|
|
allows the target process to shut down nicely, but with the
|
|
catch that the process might choose to ignore it.
|
|
|
|
Platform supporter note: only mark soft terminate as supported
|
|
if the target process has some way to evade the soft terminate
|
|
request; otherwise, just support the hard terminate method.
|
|
|
|
@return True if the platform supports a soft terminate mechanism.
|
|
"""
|
|
# By default, we do not support a soft terminate mechanism.
|
|
return False
|
|
|
|
def soft_terminate(self, popen_process, log_file=None, want_core=True):
|
|
# pylint: disable=no-self-use,unused-argument
|
|
# As expected. We want derived classes to implement this.
|
|
"""Attempts to terminate the process in a polite way.
|
|
|
|
This terminate method is intended to give the child process a
|
|
chance to clean up and exit on its own, possibly with a request
|
|
to drop a core file or equivalent (i.e. [mini-]crashdump, crashlog,
|
|
etc.) If new_process_group was set in the process creation method
|
|
and the platform supports it, this terminate call will attempt to
|
|
kill the whole process tree rooted in this child process.
|
|
|
|
@param popen_process the subprocess.Popen-like object returned
|
|
by one of the process-creation methods of this class.
|
|
|
|
@param log_file file-like object used to emit error-related
|
|
logging info. May be None if no error-related info is desired.
|
|
|
|
@param want_core True if the caller would like to get a core
|
|
dump (or the analogous crash report) from the terminated process.
|
|
"""
|
|
popen_process.terminate()
|
|
|
|
def hard_terminate(self, popen_process, log_file=None):
|
|
# pylint: disable=no-self-use,unused-argument
|
|
# As expected. We want derived classes to implement this.
|
|
"""Attempts to terminate the process immediately.
|
|
|
|
This terminate method is intended to kill child process in
|
|
a manner in which the child process has no ability to block,
|
|
and also has no ability to clean up properly. If new_process_group
|
|
was specified when creating the process, and if the platform
|
|
implementation supports it, this will attempt to kill the
|
|
whole process tree rooted in the child process.
|
|
|
|
@param popen_process the subprocess.Popen-like object returned
|
|
by one of the process-creation methods of this class.
|
|
|
|
@param log_file file-like object used to emit error-related
|
|
logging info. May be None if no error-related info is desired.
|
|
"""
|
|
popen_process.kill()
|
|
|
|
def was_soft_terminate(self, returncode, with_core):
|
|
# pylint: disable=no-self-use,unused-argument
|
|
# As expected. We want derived classes to implement this.
|
|
"""Returns if Popen-like object returncode matches soft terminate.
|
|
|
|
@param returncode the returncode from the Popen-like object that
|
|
terminated with a given return code.
|
|
|
|
@param with_core indicates whether the returncode should match
|
|
a core-generating return signal.
|
|
|
|
@return True when the returncode represents what the system would
|
|
issue when a soft_terminate() with the given with_core arg occurred;
|
|
False otherwise.
|
|
"""
|
|
if not self.supports_soft_terminate():
|
|
# If we don't support soft termination on this platform,
|
|
# then this should always be False.
|
|
return False
|
|
else:
|
|
# Once a platform claims to support soft terminate, it
|
|
# needs to be able to identify it by overriding this method.
|
|
raise Exception("platform needs to implement")
|
|
|
|
def was_hard_terminate(self, returncode):
|
|
# pylint: disable=no-self-use,unused-argument
|
|
# As expected. We want derived classes to implement this.
|
|
"""Returns if Popen-like object returncode matches that of a hard
|
|
terminate attempt.
|
|
|
|
@param returncode the returncode from the Popen-like object that
|
|
terminated with a given return code.
|
|
|
|
@return True when the returncode represents what the system would
|
|
issue when a hard_terminate() occurred; False
|
|
otherwise.
|
|
"""
|
|
raise Exception("platform needs to implement")
|
|
|
|
def soft_terminate_signals(self):
|
|
# pylint: disable=no-self-use
|
|
"""Retrieve signal numbers that can be sent to soft terminate.
|
|
@return a list of signal numbers that can be sent to soft terminate
|
|
a process, or None if not applicable.
|
|
"""
|
|
return None
|
|
|
|
def is_exceptional_exit(self, popen_status):
|
|
"""Returns whether the program exit status is exceptional.
|
|
|
|
Returns whether the return code from a Popen process is exceptional
|
|
(e.g. signals on POSIX systems).
|
|
|
|
Derived classes should override this if they can detect exceptional
|
|
program exit.
|
|
|
|
@return True if the given popen_status represents an exceptional
|
|
program exit; False otherwise.
|
|
"""
|
|
return False
|
|
|
|
def exceptional_exit_details(self, popen_status):
|
|
"""Returns the normalized exceptional exit code and a description.
|
|
|
|
Given an exceptional exit code, returns the integral value of the
|
|
exception (e.g. signal number for POSIX) and a description (e.g.
|
|
signal name on POSIX) for the result.
|
|
|
|
Derived classes should override this if they can detect exceptional
|
|
program exit.
|
|
|
|
It is fine to not implement this so long as is_exceptional_exit()
|
|
always returns False.
|
|
|
|
@return (normalized exception code, symbolic exception description)
|
|
"""
|
|
raise Exception("exception_exit_details() called on unsupported class")
|
|
|
|
|
|
class UnixProcessHelper(ProcessHelper):
|
|
"""Provides a ProcessHelper for Unix-like operating systems.
|
|
|
|
This implementation supports anything that looks Posix-y
|
|
(e.g. Darwin, Linux, *BSD, etc.)
|
|
"""
|
|
|
|
def __init__(self):
|
|
super(UnixProcessHelper, self).__init__()
|
|
|
|
@classmethod
|
|
def _create_new_process_group(cls):
|
|
"""Creates a new process group for the calling process."""
|
|
os.setpgid(os.getpid(), os.getpid())
|
|
|
|
def create_piped_process(self, command, new_process_group=True):
|
|
# Determine what to run after the fork but before the exec.
|
|
if new_process_group:
|
|
preexec_func = self._create_new_process_group
|
|
else:
|
|
preexec_func = None
|
|
|
|
# Create the process.
|
|
process = subprocess.Popen(
|
|
command,
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
universal_newlines=True, # Elicits automatic byte -> string decoding in Py3
|
|
close_fds=True,
|
|
preexec_fn=preexec_func)
|
|
|
|
# Remember whether we're using process groups for this
|
|
# process.
|
|
process.using_process_groups = new_process_group
|
|
return process
|
|
|
|
def supports_soft_terminate(self):
|
|
# POSIX does support a soft terminate via:
|
|
# * SIGTERM (no core requested)
|
|
# * SIGQUIT (core requested if enabled, see ulimit -c)
|
|
return True
|
|
|
|
@classmethod
|
|
def _validate_pre_terminate(cls, popen_process, log_file):
|
|
# Validate args.
|
|
if popen_process is None:
|
|
raise ValueError("popen_process is None")
|
|
|
|
# Ensure we have something that looks like a valid process.
|
|
if popen_process.pid < 1:
|
|
if log_file:
|
|
log_file.write("skipping soft_terminate(): no process id")
|
|
return False
|
|
|
|
# We only do the process liveness check if we're not using
|
|
# process groups. With process groups, checking if the main
|
|
# inferior process is dead and short circuiting here is no
|
|
# good - children of it in the process group could still be
|
|
# alive, and they should be killed during a timeout.
|
|
if not popen_process.using_process_groups:
|
|
# Don't kill if it's already dead.
|
|
popen_process.poll()
|
|
if popen_process.returncode is not None:
|
|
# It has a returncode. It has already stopped.
|
|
if log_file:
|
|
log_file.write(
|
|
"requested to terminate pid {} but it has already "
|
|
"terminated, returncode {}".format(
|
|
popen_process.pid, popen_process.returncode))
|
|
# Move along...
|
|
return False
|
|
|
|
# Good to go.
|
|
return True
|
|
|
|
def _kill_with_signal(self, popen_process, log_file, signum):
|
|
# Validate we're ready to terminate this.
|
|
if not self._validate_pre_terminate(popen_process, log_file):
|
|
return
|
|
|
|
# Choose kill mechanism based on whether we're targeting
|
|
# a process group or just a process.
|
|
try:
|
|
if popen_process.using_process_groups:
|
|
# if log_file:
|
|
# log_file.write(
|
|
# "sending signum {} to process group {} now\n".format(
|
|
# signum, popen_process.pid))
|
|
os.killpg(popen_process.pid, signum)
|
|
else:
|
|
# if log_file:
|
|
# log_file.write(
|
|
# "sending signum {} to process {} now\n".format(
|
|
# signum, popen_process.pid))
|
|
os.kill(popen_process.pid, signum)
|
|
except OSError as error:
|
|
import errno
|
|
if error.errno == errno.ESRCH:
|
|
# This is okay - failed to find the process. It may be that
|
|
# that the timeout pre-kill hook eliminated the process. We'll
|
|
# ignore.
|
|
pass
|
|
else:
|
|
raise
|
|
|
|
def soft_terminate(self, popen_process, log_file=None, want_core=True):
|
|
# Choose signal based on desire for core file.
|
|
if want_core:
|
|
# SIGQUIT will generate core by default. Can be caught.
|
|
signum = signal.SIGQUIT
|
|
else:
|
|
# SIGTERM is the traditional nice way to kill a process.
|
|
# Can be caught, doesn't generate a core.
|
|
signum = signal.SIGTERM
|
|
|
|
self._kill_with_signal(popen_process, log_file, signum)
|
|
|
|
def hard_terminate(self, popen_process, log_file=None):
|
|
self._kill_with_signal(popen_process, log_file, signal.SIGKILL)
|
|
|
|
def was_soft_terminate(self, returncode, with_core):
|
|
if with_core:
|
|
return returncode == -signal.SIGQUIT
|
|
else:
|
|
return returncode == -signal.SIGTERM
|
|
|
|
def was_hard_terminate(self, returncode):
|
|
return returncode == -signal.SIGKILL
|
|
|
|
def soft_terminate_signals(self):
|
|
return [signal.SIGQUIT, signal.SIGTERM]
|
|
|
|
def is_exceptional_exit(self, popen_status):
|
|
return popen_status < 0
|
|
|
|
@classmethod
|
|
def _signal_names_by_number(cls):
|
|
return dict(
|
|
(k, v) for v, k in reversed(sorted(signal.__dict__.items()))
|
|
if v.startswith('SIG') and not v.startswith('SIG_'))
|
|
|
|
def exceptional_exit_details(self, popen_status):
|
|
signo = -popen_status
|
|
signal_names_by_number = self._signal_names_by_number()
|
|
signal_name = signal_names_by_number.get(signo, "")
|
|
return (signo, signal_name)
|
|
|
|
|
|
class WindowsProcessHelper(ProcessHelper):
|
|
"""Provides a Windows implementation of the ProcessHelper class."""
|
|
|
|
def __init__(self):
|
|
super(WindowsProcessHelper, self).__init__()
|
|
|
|
def create_piped_process(self, command, new_process_group=True):
|
|
if new_process_group:
|
|
# We need this flag if we want os.kill() to work on the subprocess.
|
|
creation_flags = subprocess.CREATE_NEW_PROCESS_GROUP
|
|
else:
|
|
creation_flags = 0
|
|
|
|
return subprocess.Popen(
|
|
command,
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
universal_newlines=True, # Elicits automatic byte -> string decoding in Py3
|
|
creationflags=creation_flags)
|
|
|
|
def was_hard_terminate(self, returncode):
|
|
return returncode != 0
|
|
|
|
|
|
class ProcessDriver(object):
|
|
"""Drives a child process, notifies on important events, and can timeout.
|
|
|
|
Clients are expected to derive from this class and override the
|
|
on_process_started and on_process_exited methods if they want to
|
|
hook either of those.
|
|
|
|
This class supports timing out the child process in a platform-agnostic
|
|
way. The on_process_exited method is informed if the exit was natural
|
|
or if it was due to a timeout.
|
|
"""
|
|
|
|
def __init__(self, soft_terminate_timeout=10.0):
|
|
super(ProcessDriver, self).__init__()
|
|
self.process_helper = ProcessHelper.process_helper()
|
|
self.pid = None
|
|
# Create the synchronization event for notifying when the
|
|
# inferior dotest process is complete.
|
|
self.done_event = threading.Event()
|
|
self.io_thread = None
|
|
self.process = None
|
|
# Number of seconds to wait for the soft terminate to
|
|
# wrap up, before moving to more drastic measures.
|
|
# Might want this longer if core dumps are generated and
|
|
# take a long time to write out.
|
|
self.soft_terminate_timeout = soft_terminate_timeout
|
|
# Number of seconds to wait for the hard terminate to
|
|
# wrap up, before giving up on the io thread. This should
|
|
# be fast.
|
|
self.hard_terminate_timeout = 5.0
|
|
self.returncode = None
|
|
|
|
# =============================================
|
|
# Methods for subclasses to override if desired.
|
|
# =============================================
|
|
|
|
def on_process_started(self):
|
|
pass
|
|
|
|
def on_process_exited(self, command, output, was_timeout, exit_status):
|
|
pass
|
|
|
|
def on_timeout_pre_kill(self):
|
|
"""Called after the timeout interval elapses but before killing it.
|
|
|
|
This method is added to enable derived classes the ability to do
|
|
something to the process prior to it being killed. For example,
|
|
this would be a good spot to run a program that samples the process
|
|
to see what it was doing (or not doing).
|
|
|
|
Do not attempt to reap the process (i.e. use wait()) in this method.
|
|
That will interfere with the kill mechanism and return code processing.
|
|
"""
|
|
|
|
def write(self, content):
|
|
# pylint: disable=no-self-use
|
|
# Intended - we want derived classes to be able to override
|
|
# this and use any self state they may contain.
|
|
sys.stdout.write(content)
|
|
|
|
# ==============================================================
|
|
# Operations used to drive processes. Clients will want to call
|
|
# one of these.
|
|
# ==============================================================
|
|
|
|
def run_command(self, command):
|
|
# Start up the child process and the thread that does the
|
|
# communication pump.
|
|
self._start_process_and_io_thread(command)
|
|
|
|
# Wait indefinitely for the child process to finish
|
|
# communicating. This indicates it has closed stdout/stderr
|
|
# pipes and is done.
|
|
self.io_thread.join()
|
|
self.returncode = self.process.wait()
|
|
if self.returncode is None:
|
|
raise Exception(
|
|
"no exit status available for pid {} after the "
|
|
" inferior dotest.py should have completed".format(
|
|
self.process.pid))
|
|
|
|
# Notify of non-timeout exit.
|
|
self.on_process_exited(
|
|
command,
|
|
self.io_thread.output,
|
|
False,
|
|
self.returncode)
|
|
|
|
def run_command_with_timeout(self, command, timeout, want_core):
|
|
# Figure out how many seconds our timeout description is requesting.
|
|
timeout_seconds = timeout_to_seconds(timeout)
|
|
|
|
# Start up the child process and the thread that does the
|
|
# communication pump.
|
|
self._start_process_and_io_thread(command)
|
|
|
|
self._wait_with_timeout(timeout_seconds, command, want_core)
|
|
|
|
# ================
|
|
# Internal details.
|
|
# ================
|
|
|
|
def _start_process_and_io_thread(self, command):
|
|
# Create the process.
|
|
self.process = self.process_helper.create_piped_process(command)
|
|
self.pid = self.process.pid
|
|
self.on_process_started()
|
|
|
|
# Ensure the event is cleared that is used for signaling
|
|
# from the communication() thread when communication is
|
|
# complete (i.e. the inferior process has finished).
|
|
self.done_event.clear()
|
|
|
|
self.io_thread = CommunicatorThread(
|
|
self.process, self.done_event, self.write)
|
|
self.io_thread.start()
|
|
|
|
def _attempt_soft_kill(self, want_core):
|
|
# The inferior dotest timed out. Attempt to clean it
|
|
# with a non-drastic method (so it can clean up properly
|
|
# and/or generate a core dump). Often the OS can't guarantee
|
|
# that the process will really terminate after this.
|
|
self.process_helper.soft_terminate(
|
|
self.process,
|
|
want_core=want_core,
|
|
log_file=self)
|
|
|
|
# Now wait up to a certain timeout period for the io thread
|
|
# to say that the communication ended. If that wraps up
|
|
# within our soft terminate timeout, we're all done here.
|
|
self.io_thread.join(self.soft_terminate_timeout)
|
|
if not self.io_thread.is_alive():
|
|
# stdout/stderr were closed on the child process side. We
|
|
# should be able to wait and reap the child process here.
|
|
self.returncode = self.process.wait()
|
|
# We terminated, and the done_trying result is n/a
|
|
terminated = True
|
|
done_trying = None
|
|
else:
|
|
self.write("soft kill attempt of process {} timed out "
|
|
"after {} seconds\n".format(
|
|
self.process.pid, self.soft_terminate_timeout))
|
|
terminated = False
|
|
done_trying = False
|
|
return terminated, done_trying
|
|
|
|
def _attempt_hard_kill(self):
|
|
# Instruct the process to terminate and really force it to
|
|
# happen. Don't give the process a chance to ignore.
|
|
self.process_helper.hard_terminate(
|
|
self.process,
|
|
log_file=self)
|
|
|
|
# Reap the child process. This should not hang as the
|
|
# hard_kill() mechanism is supposed to really kill it.
|
|
# Improvement option:
|
|
# If this does ever hang, convert to a self.process.poll()
|
|
# loop checking on self.process.returncode until it is not
|
|
# None or the timeout occurs.
|
|
self.returncode = self.process.wait()
|
|
|
|
# Wait a few moments for the io thread to finish...
|
|
self.io_thread.join(self.hard_terminate_timeout)
|
|
if self.io_thread.is_alive():
|
|
# ... but this is not critical if it doesn't end for some
|
|
# reason.
|
|
self.write(
|
|
"hard kill of process {} timed out after {} seconds waiting "
|
|
"for the io thread (ignoring)\n".format(
|
|
self.process.pid, self.hard_terminate_timeout))
|
|
|
|
# Set if it terminated. (Set up for optional improvement above).
|
|
terminated = self.returncode is not None
|
|
# Nothing else to try.
|
|
done_trying = True
|
|
|
|
return terminated, done_trying
|
|
|
|
def _attempt_termination(self, attempt_count, want_core):
|
|
if self.process_helper.supports_soft_terminate():
|
|
# When soft termination is supported, we first try to stop
|
|
# the process with a soft terminate. Failing that, we try
|
|
# the hard terminate option.
|
|
if attempt_count == 1:
|
|
return self._attempt_soft_kill(want_core)
|
|
elif attempt_count == 2:
|
|
return self._attempt_hard_kill()
|
|
else:
|
|
# We don't have anything else to try.
|
|
terminated = self.returncode is not None
|
|
done_trying = True
|
|
return terminated, done_trying
|
|
else:
|
|
# We only try the hard terminate option when there
|
|
# is no soft terminate available.
|
|
if attempt_count == 1:
|
|
return self._attempt_hard_kill()
|
|
else:
|
|
# We don't have anything else to try.
|
|
terminated = self.returncode is not None
|
|
done_trying = True
|
|
return terminated, done_trying
|
|
|
|
def _wait_with_timeout(self, timeout_seconds, command, want_core):
|
|
# Allow up to timeout seconds for the io thread to wrap up.
|
|
# If that completes, the child process should be done.
|
|
completed_normally = self.done_event.wait(timeout_seconds)
|
|
if completed_normally:
|
|
# Reap the child process here.
|
|
self.returncode = self.process.wait()
|
|
else:
|
|
|
|
# Allow derived classes to do some work after we detected
|
|
# a timeout but before we touch the timed-out process.
|
|
self.on_timeout_pre_kill()
|
|
|
|
# Prepare to stop the process
|
|
process_terminated = completed_normally
|
|
terminate_attempt_count = 0
|
|
|
|
# Try as many attempts as we support for trying to shut down
|
|
# the child process if it's not already shut down.
|
|
while not process_terminated:
|
|
terminate_attempt_count += 1
|
|
# Attempt to terminate.
|
|
process_terminated, done_trying = self._attempt_termination(
|
|
terminate_attempt_count, want_core)
|
|
# Check if there's nothing more to try.
|
|
if done_trying:
|
|
# Break out of our termination attempt loop.
|
|
break
|
|
|
|
# At this point, we're calling it good. The process
|
|
# finished gracefully, was shut down after one or more
|
|
# attempts, or we failed but gave it our best effort.
|
|
self.on_process_exited(
|
|
command,
|
|
self.io_thread.output,
|
|
not completed_normally,
|
|
self.returncode)
|
|
|
|
|
|
def patched_init(self, *args, **kwargs):
|
|
self.original_init(*args, **kwargs)
|
|
# Initialize our condition variable that protects wait()/poll().
|
|
self.wait_condition = threading.Condition()
|
|
|
|
|
|
def patched_wait(self, *args, **kwargs):
|
|
self.wait_condition.acquire()
|
|
try:
|
|
result = self.original_wait(*args, **kwargs)
|
|
# The process finished. Signal the condition.
|
|
self.wait_condition.notify_all()
|
|
return result
|
|
finally:
|
|
self.wait_condition.release()
|
|
|
|
|
|
def patched_poll(self, *args, **kwargs):
|
|
self.wait_condition.acquire()
|
|
try:
|
|
result = self.original_poll(*args, **kwargs)
|
|
if self.returncode is not None:
|
|
# We did complete, and we have the return value.
|
|
# Signal the event to indicate we're done.
|
|
self.wait_condition.notify_all()
|
|
return result
|
|
finally:
|
|
self.wait_condition.release()
|
|
|
|
|
|
def patch_up_subprocess_popen():
|
|
subprocess.Popen.original_init = subprocess.Popen.__init__
|
|
subprocess.Popen.__init__ = patched_init
|
|
|
|
subprocess.Popen.original_wait = subprocess.Popen.wait
|
|
subprocess.Popen.wait = patched_wait
|
|
|
|
subprocess.Popen.original_poll = subprocess.Popen.poll
|
|
subprocess.Popen.poll = patched_poll
|
|
|
|
# Replace key subprocess.Popen() threading-unprotected methods with
|
|
# threading-protected versions.
|
|
patch_up_subprocess_popen()
|