Python psutil.TimeoutExpired() Examples

def BlockOnProcessPID(self, pid, create_time, timeout=None):
        if pid is None:
            return
        p = None
        try:
            p = self.GetProcessByPID(pid, create_time)
            if p is not None and p.is_running() and p.status not in [psutil.STATUS_ZOMBIE, psutil.STATUS_DEAD]:
                val = p.wait(timeout)
                if val is None:
                    self.logger.debug("Process is terminated ")
                else:
                    self.logger.debug("Process is terminated, possibly by os ")
        except psutil.TimeoutExpired:
            self.logger.info("TimeoutExpired happens")
            if p is not None:
                self.logger.info("kill process")
                self.TerminateProcessTree(pid, create_time)
        except Exception:
            self.logger.debug("Process is terminated for unknown reasons")
            pass
        return 

def kill_process(process):
    """Kill a single process.

    Args:
        process (psutil.Process): process to kill.
    """
    if not process.is_running():
        return

    process.kill()

    try:
        process.wait(PROCESS_TERMINATION_TIMEOUT)

    except psutil.TimeoutExpired:
        core_log.warning("Process %d failed to terminate", process.pid) 

def test_wait_timeout_0(self):
        sproc = get_test_subprocess()
        p = psutil.Process(sproc.pid)
        self.assertRaises(psutil.TimeoutExpired, p.wait, 0)
        p.kill()
        stop_at = time.time() + 2
        while True:
            try:
                code = p.wait(0)
            except psutil.TimeoutExpired:
                if time.time() >= stop_at:
                    raise
            else:
                break
        if POSIX:
            self.assertEqual(code, -signal.SIGKILL)
        else:
            self.assertEqual(code, signal.SIGTERM)
        self.assertFalse(p.is_running()) 

def test_wait_non_children(self):
        # Test wait() against a process which is not our direct
        # child.
        p1, p2 = create_proc_children_pair()
        self.assertRaises(psutil.TimeoutExpired, p1.wait, 0.01)
        self.assertRaises(psutil.TimeoutExpired, p2.wait, 0.01)
        # We also terminate the direct child otherwise the
        # grandchild will hang until the parent is gone.
        p1.terminate()
        p2.terminate()
        ret1 = p1.wait()
        ret2 = p2.wait()
        if POSIX:
            self.assertEqual(ret1, -signal.SIGTERM)
            # For processes which are not our children we're supposed
            # to get None.
            self.assertEqual(ret2, None)
        else:
            self.assertEqual(ret1, signal.SIGTERM)
            self.assertEqual(ret1, signal.SIGTERM) 

def exists(pid, time, delta_seconds=3, zombie_seconds=10):
    if not ps.pid_exists(pid):
        log.debug(f'Process {pid} does not exist')
        return False
    process = ps.Process(pid)
    if process.status() == ps.STATUS_ZOMBIE:
        try:
            log.warning(f'Waiting for zombie {pid}')
            process.wait(zombie_seconds)
        except ps.TimeoutExpired:
            log.debug(f'Timeout waiting for zombie {pid}')
        return False
    elif delta_seconds:
        creation_ok = abs(process.create_time() - time.timestamp()) < delta_seconds
        if creation_ok:
            log.debug(f'Process {pid} still exists')
        else:
            log.debug(f'Creation time for process {pid} incorrect ({process.create_time()} / {time.timestamp()})')
        return creation_ok
    else:
        log.debug(f'Assuming process {pid} is same as expected')
        return True   # if delta_seconds = 0 don't check, just assume same 

def terminate(self):
        # return codes:
        # 0   no work to do
        # 1   terminated
        # -1  failed to terminate

        if not self.pid or not self.isrunning():
            return 0

        timeout = 20

        try:
            proc = psutil.Process(self.pid)
            proc.terminate()
            try:
                proc.wait(timeout=timeout/2)
            except psutil.TimeoutExpired:
                proc.kill()
                proc.wait(timeout=timeout/2)
        except psutil.Error:
            return -1

        return 1 

def stop_tasks(dr):
    logger.debug(f"""Stop tasks for {dr.dag_id} - {dr.run_id}""")
    for ti in dr.get_task_instances():
        logger.debug(f"""process {ti.dag_id} - {ti.task_id} - {ti.execution_date} - {ti.pid}""")
        if ti.state == State.RUNNING:
            try:
                process = psutil.Process(ti.pid) if ti.pid else None
            except Exception:
                logger.debug(f" - cannot find process by PID {ti.pid}")
                process = None
            ti.set_state(State.FAILED)
            logger.debug(" - set state to FAILED")
            if process:
                logger.debug(f" - wait for process {ti.pid} to exit")
                try:
                    process.wait(timeout=TIMEOUT * 2)  # raises psutil.TimeoutExpired if timeout. Makes task fail -> DagRun fails
                except psutil.TimeoutExpired as e:
                    logger.debug(f" - Done waiting for process {ti.pid} to die") 

def stop_tasks(self, dr):
        logger.debug(f"""Stop tasks for {dr.dag_id} - {dr.run_id}""")
        for ti in dr.get_task_instances():
            logger.debug(f"""process {ti.dag_id} - {ti.task_id} - {ti.execution_date} - {ti.pid}""")
            if ti.state == State.RUNNING:
                try:
                    process = psutil.Process(ti.pid) if ti.pid else None
                except Exception:
                    logger.debug(f" - cannot find process by PID {ti.pid}")
                    process = None
                ti.set_state(State.FAILED)
                logger.debug(" - set state to FAILED")
                if process:
                    logger.debug(f" - wait for process {ti.pid} to exit")
                    try:
                        process.wait(timeout=TIMEOUT * 2)  # raises psutil.TimeoutExpired if timeout. Makes task fail -> DagRun fails
                    except psutil.TimeoutExpired as e:
                        logger.debug(f" - Done waiting for process {ti.pid} to die") 

def _attend_process(self, proc, sleeptime):
        """
        Waits on a process for a given time to see if it finishes, returns True
        if it's still running after the given time or False as soon as it 
        returns.

        :param psutil.Popen proc: Process object opened by psutil.Popen()
        :param float sleeptime: Time to wait
        :return bool: True if process is still running; otherwise false
        """
        # print("attend:{}".format(proc.pid))
        try:
            proc.wait(timeout=sleeptime)
        except psutil.TimeoutExpired:
            return True
        return False 

def _wait_output(popen, is_slow):
    """Returns `True` if we can get output of the command in the
    `settings.wait_command` time.

    Command will be killed if it wasn't finished in the time.

    :type popen: Popen
    :rtype: bool

    """
    proc = Process(popen.pid)
    try:
        proc.wait(settings.wait_slow_command if is_slow
                  else settings.wait_command)
        return True
    except TimeoutExpired:
        for child in proc.children(recursive=True):
            _kill_process(child)
        _kill_process(proc)
        return False 

def test_launcher():
    # proc = multiprocessing.Process(target=Launcher().launch)
    # proc.start()
    # time.sleep(2)
    # module_procs = psutil.Process(proc.pid).children(recursive=True)

    # assert len(module_procs) > 2
    # proc.terminate()
    # proc.join(5)
    # if proc.is_alive():
    #     raise Exception(
    #         "launcher process with pid {} may not be killed success!")

    # time.sleep(Config().get_config("monitor.schedule_interval") * 2)
    # for child in module_procs:
    #     try:
    #         child.wait(5)
    #     except psutil.TimeoutExpired:
    #         assert False

    Communicator.reset() 

def preprocessor_fixture():
    helper.reset_db()
    Communicator()
    Logger()
    module_proc = modules.Process(modules.Preprocessor)
    module_proc.start()

    yield module_proc

    root_proc = psutil.Process(module_proc.pid)
    procs = root_proc.children(recursive=True)
    try:
        root_proc.terminate()
        root_proc.wait(10)
        module_proc.join(5)
        for p in procs:
            p.terminate()
            p.wait(10)
    except psutil.TimeoutExpired:
        raise Exception("Module process may not be killed success!")

    helper.reset_db()
    Communicator.reset() 

def test_wait_non_children(self):
        # Test wait() against a process which is not our direct
        # child.
        p1, p2 = create_proc_children_pair()
        self.assertRaises(psutil.TimeoutExpired, p1.wait, 0.01)
        self.assertRaises(psutil.TimeoutExpired, p2.wait, 0.01)
        # We also terminate the direct child otherwise the
        # grandchild will hang until the parent is gone.
        p1.terminate()
        p2.terminate()
        ret1 = p1.wait()
        ret2 = p2.wait()
        if POSIX:
            self.assertEqual(ret1, -signal.SIGTERM)
            # For processes which are not our children we're supposed
            # to get None.
            self.assertEqual(ret2, None)
        else:
            self.assertEqual(ret1, signal.SIGTERM)
            self.assertEqual(ret1, signal.SIGTERM) 

def test_wait_timeout_0(self):
        sproc = get_test_subprocess()
        p = psutil.Process(sproc.pid)
        self.assertRaises(psutil.TimeoutExpired, p.wait, 0)
        p.kill()
        stop_at = time.time() + 2
        while True:
            try:
                code = p.wait(0)
            except psutil.TimeoutExpired:
                if time.time() >= stop_at:
                    raise
            else:
                break
        if POSIX:
            self.assertEqual(code, -signal.SIGKILL)
        else:
            self.assertEqual(code, signal.SIGTERM)
        self.assertFalse(p.is_running()) 

def test_wait_timeout_nonblocking(self):
        p = self.spawn_psproc()
        self.assertRaises(psutil.TimeoutExpired, p.wait, 0)
        p.kill()
        stop_at = time.time() + GLOBAL_TIMEOUT
        while time.time() < stop_at:
            try:
                code = p.wait(0)
                break
            except psutil.TimeoutExpired:
                pass
        else:
            raise self.fail('timeout')
        if POSIX:
            self.assertEqual(code, -signal.SIGKILL)
        else:
            self.assertEqual(code, signal.SIGTERM)
        self.assertProcessGone(p) 

def test_wait_non_children(self):
        # Test wait() against a process which is not our direct
        # child.
        child, grandchild = self.spawn_children_pair()
        self.assertRaises(psutil.TimeoutExpired, child.wait, 0.01)
        self.assertRaises(psutil.TimeoutExpired, grandchild.wait, 0.01)
        # We also terminate the direct child otherwise the
        # grandchild will hang until the parent is gone.
        child.terminate()
        grandchild.terminate()
        child_ret = child.wait()
        grandchild_ret = grandchild.wait()
        if POSIX:
            self.assertEqual(child_ret, -signal.SIGTERM)
            # For processes which are not our children we're supposed
            # to get None.
            self.assertEqual(grandchild_ret, None)
        else:
            self.assertEqual(child_ret, signal.SIGTERM)
            self.assertEqual(child_ret, signal.SIGTERM) 

def test_wait_stopped(self):
        p = self.spawn_psproc()
        if POSIX:
            # Test waitpid() + WIFSTOPPED and WIFCONTINUED.
            # Note: if a process is stopped it ignores SIGTERM.
            p.send_signal(signal.SIGSTOP)
            self.assertRaises(psutil.TimeoutExpired, p.wait, timeout=0.001)
            p.send_signal(signal.SIGCONT)
            self.assertRaises(psutil.TimeoutExpired, p.wait, timeout=0.001)
            p.send_signal(signal.SIGTERM)
            self.assertEqual(p.wait(), -signal.SIGTERM)
            self.assertEqual(p.wait(), -signal.SIGTERM)
        else:
            p.suspend()
            self.assertRaises(psutil.TimeoutExpired, p.wait, timeout=0.001)
            p.resume()
            self.assertRaises(psutil.TimeoutExpired, p.wait, timeout=0.001)
            p.terminate()
            self.assertEqual(p.wait(), signal.SIGTERM)
            self.assertEqual(p.wait(), signal.SIGTERM) 

def return_code(self, timeout=0):
        # We call this multiple times, but we can only wait on the process once
        if self._rc is not None or not self.process:
            return self._rc

        try:
            self._rc = self.process.wait(timeout=timeout)
            self.process = None
        except psutil.TimeoutExpired:
            pass

        return self._rc 

def test_timeout_expired__repr__(self, func=repr):
        self.assertEqual(
            repr(psutil.TimeoutExpired(321)),
            "psutil.TimeoutExpired timeout after 321 seconds")
        self.assertEqual(
            repr(psutil.TimeoutExpired(321, pid=111)),
            "psutil.TimeoutExpired timeout after 321 seconds (pid=111)")
        self.assertEqual(
            repr(psutil.TimeoutExpired(321, pid=111, name='foo')),
            "psutil.TimeoutExpired timeout after 321 seconds "
            "(pid=111, name='foo')") 

def test_stop(self, mock_is_running):
        # Postmaster is not running
        mock_callback = Mock()
        mock_is_running.return_value = None
        self.assertTrue(self.p.stop(on_safepoint=mock_callback))
        mock_callback.assert_called()

        # Is running, stopped successfully
        mock_is_running.return_value = mock_postmaster = MockPostmaster()
        mock_callback.reset_mock()
        self.assertTrue(self.p.stop(on_safepoint=mock_callback))
        mock_callback.assert_called()
        mock_postmaster.signal_stop.assert_called()

        # Timed out waiting for fast shutdown triggers immediate shutdown
        mock_postmaster.wait.side_effect = [psutil.TimeoutExpired(30), psutil.TimeoutExpired(30), Mock()]
        mock_callback.reset_mock()
        self.assertTrue(self.p.stop(on_safepoint=mock_callback, stop_timeout=30))
        mock_callback.assert_called()
        mock_postmaster.signal_stop.assert_called()

        # Immediate shutdown succeeded
        mock_postmaster.wait.side_effect = [psutil.TimeoutExpired(30), Mock()]
        self.assertTrue(self.p.stop(on_safepoint=mock_callback, stop_timeout=30))

        # Stop signal failed
        mock_postmaster.signal_stop.return_value = False
        self.assertFalse(self.p.stop())

        # Stop signal failed to find process
        mock_postmaster.signal_stop.return_value = True
        mock_callback.reset_mock()
        self.assertTrue(self.p.stop(on_safepoint=mock_callback))
        mock_callback.assert_called()

        # Fast shutdown is timed out but when immediate postmaster is already gone
        mock_postmaster.wait.side_effect = [psutil.TimeoutExpired(30), Mock()]
        mock_postmaster.signal_stop.side_effect = [None, True]
        self.assertTrue(self.p.stop(on_safepoint=mock_callback, stop_timeout=30)) 

def terminate_postmaster(self, postmaster, mode, stop_timeout):
        if mode in ['fast', 'smart']:
            try:
                success = postmaster.signal_stop('immediate', self.pgcommand('pg_ctl'))
                if success:
                    return True
                postmaster.wait(timeout=stop_timeout)
                return True
            except TimeoutExpired:
                pass
        logger.warning("Sending SIGKILL to Postmaster and its children")
        return postmaster.signal_kill() 

def _do_stop(self, mode, block_callbacks, checkpoint, on_safepoint, stop_timeout):
        postmaster = self.is_running()
        if not postmaster:
            if on_safepoint:
                on_safepoint()
            return True, False

        if checkpoint and not self.is_starting():
            self.checkpoint(timeout=stop_timeout)

        if not block_callbacks:
            self.set_state('stopping')

        # Send signal to postmaster to stop
        success = postmaster.signal_stop(mode, self.pgcommand('pg_ctl'))
        if success is not None:
            if success and on_safepoint:
                on_safepoint()
            return success, True

        # We can skip safepoint detection if we don't have a callback
        if on_safepoint:
            # Wait for our connection to terminate so we can be sure that no new connections are being initiated
            self._wait_for_connection_close(postmaster)
            postmaster.wait_for_user_backends_to_close()
            on_safepoint()

        try:
            postmaster.wait(timeout=stop_timeout)
        except TimeoutExpired:
            logger.warning("Timeout during postmaster stop, aborting Postgres.")
            if not self.terminate_postmaster(postmaster, mode, stop_timeout):
                postmaster.wait()

        return True, True 

def send_signal(proc, sig, timeout=DEFAULT_TIMEOUT):
    # type: (psutil.Process, enum.IntEnum, int) -> bool
    """
    Sends signal `sig` to process `proc`, waiting on each and handles timeouts
    as well as nonexistent pids. Returns whether pid was successfully sent
    signal.
    """
    # Don't accidentally signal core system processes.
    pid = proc.pid
    if pid < MIN_PID:
        logger.warning("Refusing to kill pid %d", pid)
        return False

    signame = str(sig).split(".")[-1]
    logger.info("Sending signal %s to pid %d", signame, pid)
    try:
        proc.send_signal(sig)
        proc.wait(timeout=timeout)
    except psutil.NoSuchProcess:
        logger.debug("Pid %d does not exist", pid)
        return False
    except psutil.TimeoutExpired:
        logger.debug("Timed out after %ds waiting for %d", timeout, pid)
        return False

    return True 

def kill_spinning_rpm_query_processes(
        self, kill_after_seconds=3600, kill_timeout=5
    ):
        # type: (int, int) -> None
        """
        Find and kill any rpm query processes over an hour old by looking explicitly for
        `rpm -q`.
        """
        for proc in psutil.process_iter():
            try:
                cmd = proc.cmdline()
                # Valid command
                if not cmd or len(cmd) < 2:
                    continue
                # Looks like rpm
                if not (re.match(r"(/(usr/)?bin/)?rpm", cmd[0]) and "-q" in cmd):
                    continue

                self.logger.info("Considering pid %s", proc.pid)
                ctime = proc.create_time()

                if time.time() - ctime > kill_after_seconds:
                    self.logger.error(
                        "Found stale rpm process: (%d) %s", proc.pid, " ".join(cmd)
                    )
                    proc.send_signal(signal.SIGKILL)
                    proc.wait(timeout=kill_timeout)

            except psutil.NoSuchProcess:
                self.logger.warning("Skipping pid %d, it disappeared", proc.pid)
                continue
            except psutil.AccessDenied:
                self.logger.warning("Skipping pid %d, cannot access it", proc.pid)
                continue
            except psutil.TimeoutExpired:
                self.logger.warning(
                    "Timed out after %ds waiting for pid %d", kill_timeout, proc.pid
                ) 

def test_run_command_with_timeout_expired(self):
        with self.assertRaises(psutil.TimeoutExpired):
            _ = utils.run_command(self.sleep_cmd(4), timeout=0.01) 

def wait_for_standing_subprocess(proc, timeout=None):
    """Waits for a subprocess started by start_standing_subprocess to finish
    or times out.

    Propagates the exception raised by the subprocess.wait(.) function.
    The subprocess.TimeoutExpired exception is raised if the process timed-out
    rather than terminating.

    If no exception is raised: the subprocess terminated on its own. No need
    to call stop_standing_subprocess() to kill it.

    If an exception is raised: the subprocess is still alive - it did not
    terminate. Either call stop_standing_subprocess() to kill it, or call
    wait_for_standing_subprocess() to keep waiting for it to terminate on its
    own.

    If the corresponding subprocess command generates a large amount of output
    and this method is called with a timeout value, then the command can hang
    indefinitely. See http://go/pylib/subprocess.html#subprocess.Popen.wait

    This function does not support Python 2.

    Args:
        p: Subprocess to wait for.
        timeout: An integer number of seconds to wait before timing out.
    """
    proc.wait(timeout) 

def test_timeout_expired__repr__(self, func=repr):
        self.assertEqual(
            repr(psutil.TimeoutExpired(321)),
            "psutil.TimeoutExpired timeout after 321 seconds")
        self.assertEqual(
            repr(psutil.TimeoutExpired(321, pid=111)),
            "psutil.TimeoutExpired timeout after 321 seconds (pid=111)")
        self.assertEqual(
            repr(psutil.TimeoutExpired(321, pid=111, name='foo')),
            "psutil.TimeoutExpired timeout after 321 seconds "
            "(pid=111, name='foo')") 

def _pid_is_alive(cls, pid, timeout):
        """Check if a PID is alive with a timeout."""
        try:
            proc = psutil.Process(pid)
        except psutil.NoSuchProcess:
            return False

        try:
            proc.wait(timeout=timeout)
        except psutil.TimeoutExpired:
            return True

        return False 

def _terminate_process_list(process_list, kill=False, slow_stop=False):
    log.info(
        'Terminating process list:\n%s',
        pprint.pformat([_get_cmdline(proc) for proc in process_list])
    )
    for process in process_list[:]:  # Iterate over copy of the list
        if not psutil.pid_exists(process.pid):
            process_list.remove(process)
            continue
        try:
            if not kill and process.status() == psutil.STATUS_ZOMBIE:
                # Zombie processes will exit once child processes also exit
                continue
            if kill:
                log.info('Killing process(%s): %s', process.pid, _get_cmdline(process))
                process.kill()
            else:
                log.info('Terminating process(%s): %s', process.pid, _get_cmdline(process))
                try:
                    if slow_stop:
                        # Allow coverage data to be written down to disk
                        process.send_signal(signal.SIGTERM)
                        try:
                            process.wait(2)
                        except psutil.TimeoutExpired:
                            if psutil.pid_exists(process.pid):
                                continue
                    else:
                        process.terminate()
                except OSError as exc:
                    if exc.errno not in (errno.ESRCH, errno.EACCES):
                        raise
            if not psutil.pid_exists(process.pid):
                process_list.remove(process)
        except psutil.NoSuchProcess:
            process_list.remove(process) 

def wait(self, timeout=None):
        raise psutil.TimeoutExpired(timeout)