Python subprocess.TimeoutExpired() Examples

def _run(*args, env=None, check=False, timeout=None):
    encoded_args = [a.encode('utf-8') for a in args] if sys.platform != 'win32' else args
    with subprocess.Popen(encoded_args, env=env, stdout=PIPE, stderr=PIPE) as process:
        try:
            stdout, stderr = process.communicate(input, timeout=timeout)
        except TimeoutExpired:
            process.kill()
            stdout, stderr = process.communicate()
            raise TimeoutExpired(
                process.args, timeout, output=stdout, stderr=stderr,
            )
        except Exception:
            process.kill()
            process.wait()
            raise
        retcode = process.poll()
        if check and retcode:
            raise subprocess.CalledProcessError(
                retcode, process.args, output=stdout, stderr=stderr,
            )
        return subprocess.CompletedProcess(process.args, retcode, stdout, stderr) 

def run_command(args, asynchronous=False):
    """Executes a command returning its exit code and output."""
    logging.info("Executing %s command %s.",
                 asynchronous and 'asynchronous' or 'synchronous', args)

    process = subprocess.Popen(args,
                               shell=True,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.STDOUT)

    try:
        timeout = asynchronous and 1 or None
        output = process.communicate(timeout=timeout)[0].decode('utf8')
    except subprocess.TimeoutExpired:
        pass

    if asynchronous:
        return PopenOutput(None, 'Asynchronous call.')
    else:
        return PopenOutput(process.returncode, output) 

def main(_):
  shard_urls = fetch.get_urls_for_shard(FLAGS.urls_dir, FLAGS.shard_id)
  num_groups = int(math.ceil(len(shard_urls) / fetch.URLS_PER_CLIENT))
  tf.logging.info("Launching get_references_web_single_group sequentially for "
                  "%d groups in shard %d. Total URLs: %d",
                  num_groups, FLAGS.shard_id, len(shard_urls))
  command_prefix = FLAGS.command.split() + [
      "--urls_dir=%s" % FLAGS.urls_dir,
      "--shard_id=%d" % FLAGS.shard_id,
      "--debug_num_urls=%d" % FLAGS.debug_num_urls,
  ]
  with utils.timing("all_groups_fetch"):
    for i in range(num_groups):
      command = list(command_prefix)
      out_dir = os.path.join(FLAGS.out_dir, "process_%d" % i)
      command.append("--out_dir=%s" % out_dir)
      command.append("--group_id=%d" % i)
      try:
        # Even on 1 CPU, each group should finish within an hour.
        sp.check_call(command, timeout=60*60)
      except sp.TimeoutExpired:
        tf.logging.error("Group %d timed out", i) 

def test_reloader_live(runargs, mode):
    with TemporaryDirectory() as tmpdir:
        filename = os.path.join(tmpdir, "reloader.py")
        text = write_app(filename, **runargs)
        proc = Popen(argv[mode], cwd=tmpdir, stdout=PIPE, creationflags=flags)
        try:
            timeout = Timer(5, terminate, [proc])
            timeout.start()
            # Python apparently keeps using the old source sometimes if
            # we don't sleep before rewrite (pycache timestamp problem?)
            sleep(1)
            line = scanner(proc)
            assert text in next(line)
            # Edit source code and try again
            text = write_app(filename, **runargs)
            assert text in next(line)
        finally:
            timeout.cancel()
            terminate(proc)
            with suppress(TimeoutExpired):
                proc.wait(timeout=3) 

def _exec_global(self, search_args, context, input=None):
        command = ['global', '-q'] + search_args
        global_proc = subprocess.Popen(
            command,
            cwd=context['path'],
            universal_newlines=True,
            stdin=subprocess.PIPE if input else None,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
        try:
            output, error = global_proc.communicate(input=input, timeout=15)
        except subprocess.TimeoutExpired:
            global_proc.kill()
            output, error = global_proc.communicate()
        global_exitcode = global_proc.returncode

        if global_exitcode != 0:
            self._print_global_error(global_exitcode, error)
            return []

        return [t for t in output.split('\n') if len(t) > 0] 

def execute(d):
  with open("sample.bin", "wb") as f:
    f.write(d)
  
  try:
    cp = subprocess.run(
      [
          "gdb.exe", "--batch",  "--return-child-result", "-x", "fuzz.gdb",
          BINARY_PATH
      ],
      stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True,
      timeout=2.0)
  except subprocess.TimeoutExpired as e:
    sys.stdout.write("T")
    return (False, "")

  report = ""
  magic = "------------------------------"
  crashed = cp.returncode >= 0xc0000000
  if crashed:
    #print(cp.stdout)
    report = cp.stdout.split(magic)[1]

  return (crashed, report) 

def start_or_restart_ffmpeg(ffmpeg_cmd, frame_size, ffmpeg_process=None):
    if not ffmpeg_process is None:
        print("Terminating the existing ffmpeg process...")
        ffmpeg_process.terminate()
        try:
            print("Waiting for ffmpeg to exit gracefully...")
            ffmpeg_process.communicate(timeout=30)
        except sp.TimeoutExpired:
            print("FFmpeg didnt exit. Force killing...")
            ffmpeg_process.kill()
            ffmpeg_process.communicate()
        ffmpeg_process = None

    print("Creating ffmpeg process...")
    print(" ".join(ffmpeg_cmd))
    process = sp.Popen(ffmpeg_cmd, stdout = sp.PIPE, stdin = sp.DEVNULL, bufsize=frame_size*10, start_new_session=True)
    return process 

def do_subprocess(cmd, stdout=True, stderr=True, check=True, timeout=2):
    assert sys.version_info >= (3,5)
    kwargs = dict(
        stdout=subprocess.PIPE if stdout else None,
        stderr=subprocess.PIPE if stderr else None,
        shell=True,
        check=False,
        timeout=timeout,
    )

    try:
        result = subprocess.run(cmd, **kwargs)
    except subprocess.TimeoutExpired:
        msg = "Command '{}' timed out after waiting for {} seconds"
        raise ClickException(msg.format(cmd, timeout))

    if check and result.returncode:
        errno = result.returncode
        error = result.stderr.decode().strip()
        msg = "Command '{}' returned non-zero exit status {}\n{}"
        raise ClickException(msg.format(cmd, errno, error))

    return result 

def _wait_for_containers_exit_status(self, containers):
        """Wait for all of the specified containers to exit
        and return their exit codes.
        Args:
            containers (list of str): The containers to wait to exit.

        Returns:
            list of int: The list of return codes for the process in each
                container.

        """
        wait = ['docker', 'wait'] + containers
        handle = subprocess.Popen(
            args=wait,
            stdout=subprocess.PIPE,
            universal_newlines=True)
        try:
            output, _ = handle.communicate(timeout=35)
            return [int(e) for e in output.strip().split('\n')]
        except subprocess.TimeoutExpired:
            handle.kill()
            LOGGER.warning("Docker timed out waiting for %s to exit",
                           containers)
            return [] 

def runpytest_subprocess(self, *args, timeout=None):
        """Run pytest as a subprocess with given arguments.

        Any plugins added to the :py:attr:`plugins` list will be added using the
        ``-p`` command line option.  Additionally ``--basetemp`` is used to put
        any temporary files and directories in a numbered directory prefixed
        with "runpytest-" to not conflict with the normal numbered pytest
        location for temporary files and directories.

        :param args: the sequence of arguments to pass to the pytest subprocess
        :param timeout: the period in seconds after which to timeout and raise
            :py:class:`Testdir.TimeoutExpired`

        Returns a :py:class:`RunResult`.
        """
        __tracebackhide__ = True
        p = py.path.local.make_numbered_dir(
            prefix="runpytest-", keep=None, rootdir=self.tmpdir
        )
        args = ("--basetemp=%s" % p,) + args
        plugins = [x for x in self.plugins if isinstance(x, str)]
        if plugins:
            args = ("-p", plugins[0]) + args
        args = self._getpytestargs() + args
        return self.run(*args, timeout=timeout) 

def runpytest_subprocess(self, *args, timeout=None):
        """Run pytest as a subprocess with given arguments.

        Any plugins added to the :py:attr:`plugins` list will be added using the
        ``-p`` command line option.  Additionally ``--basetemp`` is used to put
        any temporary files and directories in a numbered directory prefixed
        with "runpytest-" to not conflict with the normal numbered pytest
        location for temporary files and directories.

        :param args: the sequence of arguments to pass to the pytest subprocess
        :param timeout: the period in seconds after which to timeout and raise
            :py:class:`Testdir.TimeoutExpired`

        Returns a :py:class:`RunResult`.
        """
        __tracebackhide__ = True
        p = py.path.local.make_numbered_dir(
            prefix="runpytest-", keep=None, rootdir=self.tmpdir
        )
        args = ("--basetemp=%s" % p,) + args
        plugins = [x for x in self.plugins if isinstance(x, str)]
        if plugins:
            args = ("-p", plugins[0]) + args
        args = self._getpytestargs() + args
        return self.run(*args, timeout=timeout) 

def tearDown(self, *args, **kwargs):
        if self.cli_process is not None:
            try:
                self.cli_process.terminate()
                self.cli_process.wait(5)
            except TimeoutExpired:
                try:
                    self.cli_process.kill()
                    self.cli_process.wait(5)
                except Exception as e:
                    logging.critical("cannot stop subprocess {}: {}".format(self.cli_process, e))

            if self.cli_process.returncode != 0:
                self.fail("subprocess {} returned exit code {}".format(' '.join(self.cli_args), self.cli_process.returncode))

        if self.stdout_reader_thread is not None:
            self.stdout_reader_thread.join(5)
            if self.stdout_reader_thread.is_alive():
                logging.error("reader thread not stopping...")

        if self.stderr_reader_thread is not None:
            self.stderr_reader_thread.join(5)
            if self.stderr_reader_thread.is_alive():
                logging.error("reader thread not stopping...") 

def main(_):
  shard_urls = fetch.get_urls_for_shard(FLAGS.urls_dir, FLAGS.shard_id)
  num_groups = int(math.ceil(len(shard_urls) / fetch.URLS_PER_CLIENT))
  tf.logging.info("Launching get_references_web_single_group sequentially for "
                  "%d groups in shard %d. Total URLs: %d",
                  num_groups, FLAGS.shard_id, len(shard_urls))
  command_prefix = FLAGS.command.split() + [
      "--urls_dir=%s" % FLAGS.urls_dir,
      "--shard_id=%d" % FLAGS.shard_id,
      "--debug_num_urls=%d" % FLAGS.debug_num_urls,
  ]
  with utils.timing("all_groups_fetch"):
    for i in range(num_groups):
      command = list(command_prefix)
      out_dir = os.path.join(FLAGS.out_dir, "process_%d" % i)
      command.append("--out_dir=%s" % out_dir)
      command.append("--group_id=%d" % i)
      try:
        # Even on 1 CPU, each group should finish within an hour.
        sp.check_call(command, timeout=60*60)
      except sp.TimeoutExpired:
        tf.logging.error("Group %d timed out", i) 

def main(_):
  shard_urls = fetch.get_urls_for_shard(FLAGS.urls_dir, FLAGS.shard_id)
  num_groups = int(math.ceil(len(shard_urls) / fetch.URLS_PER_CLIENT))
  tf.logging.info("Launching get_references_web_single_group sequentially for "
                  "%d groups in shard %d. Total URLs: %d",
                  num_groups, FLAGS.shard_id, len(shard_urls))
  command_prefix = FLAGS.command.split() + [
      "--urls_dir=%s" % FLAGS.urls_dir,
      "--shard_id=%d" % FLAGS.shard_id,
      "--debug_num_urls=%d" % FLAGS.debug_num_urls,
  ]
  with utils.timing("all_groups_fetch"):
    for i in range(num_groups):
      command = list(command_prefix)
      out_dir = os.path.join(FLAGS.out_dir, "process_%d" % i)
      command.append("--out_dir=%s" % out_dir)
      command.append("--group_id=%d" % i)
      try:
        # Even on 1 CPU, each group should finish within an hour.
        sp.check_call(command, timeout=60*60)
      except sp.TimeoutExpired:
        tf.logging.error("Group %d timed out", i) 

def run(self):
        try:
            script_output = subprocess.check_output(" ".join(self.gdb_cmd), shell=True, stderr=subprocess.DEVNULL,
                                                    stdin=subprocess.DEVNULL)
        except (subprocess.TimeoutExpired, subprocess.CalledProcessError) as e:
            script_output = e.output

        script_output = script_output.decode(errors='replace').splitlines()

        for line in script_output:
            matching = [line.replace(g, '') for g in self.grep_for if g in line]
            matching = " ".join(matching).strip('\' ')
            matching = matching.replace(self.out_dir, '')
            if len(matching) > 0:
                self.out_queue_lock.acquire()
                self.out_queue.put(matching)
                self.out_queue_lock.release() 

def check_screen():
    inside_screen = False
    # try:
    #     screen_output = subprocess.check_output("screen -ls", shell=True, stderr=subprocess.DEVNULL,
    #                                             universal_newlines=True)
    # except (subprocess.TimeoutExpired, subprocess.CalledProcessError) as e:
    #     screen_output = e.output
    #
    # screen_output = screen_output.splitlines()
    #
    # for line in screen_output:
    #     if "(Attached)" in line:
    #         inside_screen = True

    if os.environ.get("STY"):
        inside_screen = True

    return inside_screen 

def shutdown(self, grace_time=10) -> bool:
        """
        Tru to stop within <gracetime> seconds the current processus/worker
        :param grace_time: maximum waiting time
        :return: True if stopped, False otherwise
        """
        self.logger.info(f"Trying to stop (terminate) {self.process}")
        try:
            self.process.terminate()
            time.sleep(0.2)
        finally:
            try:
                self.process.wait(timeout=grace_time)
            except subprocess.TimeoutExpired:
                self.logger.info(f"Processus {self.process} did not terminate in time. Trying to kill it.")
            finally:
                try:
                    self.process.kill()
                    self.logger.info(f"Processus exited with {self.process.returncode}")
                    return True
                except subprocess.TimeoutExpired:
                    self.logger.info(f"Processus {self.process} is still alive .. Don't know how to stop it.")
                    return False 

def main(_):
  shard_urls = fetch.get_urls_for_shard(FLAGS.urls_dir, FLAGS.shard_id)
  num_groups = int(math.ceil(len(shard_urls) / fetch.URLS_PER_CLIENT))
  tf.logging.info("Launching get_references_web_single_group sequentially for "
                  "%d groups in shard %d. Total URLs: %d",
                  num_groups, FLAGS.shard_id, len(shard_urls))
  command_prefix = FLAGS.command.split() + [
      "--urls_dir=%s" % FLAGS.urls_dir,
      "--shard_id=%d" % FLAGS.shard_id,
      "--debug_num_urls=%d" % FLAGS.debug_num_urls,
  ]
  with utils.timing("all_groups_fetch"):
    for i in range(num_groups):
      command = list(command_prefix)
      out_dir = os.path.join(FLAGS.out_dir, "process_%d" % i)
      command.append("--out_dir=%s" % out_dir)
      command.append("--group_id=%d" % i)
      try:
        # Even on 1 CPU, each group should finish within an hour.
        sp.check_call(command, timeout=60*60)
      except sp.TimeoutExpired:
        tf.logging.error("Group %d timed out", i) 

def _run_vsm(self, cmd, input_data, sig_num_path, wait_time_ms):
        data = (input_data + self.quit_command).encode('utf8')

        timeout_s = 2
        if wait_time_ms > 0:
            timeout_s = wait_time_ms / 1000

        process = Popen(cmd, stdin=PIPE, stdout=PIPE)

        try:
            output, _ = process.communicate(data, timeout_s)
        except TimeoutExpired:
            process.kill()
            return None

        cmd_output = output.decode()

        return _remove_timestamp(cmd_output) 

def _stop_proc(self):
        """ stop the exiftool process if it's running, otherwise, do nothing """
        if not self._process_running:
            logging.warning("exiftool process is not running")
            return

        self._process.stdin.write(b"-stay_open\n")
        self._process.stdin.write(b"False\n")
        self._process.stdin.flush()
        try:
            self._process.communicate(timeout=5)
        except subprocess.TimeoutExpired:
            logging.warning(
                f"exiftool pid {self._process.pid} did not exit, killing it"
            )
            self._process.kill()
            self._process.communicate()

        del self._process
        self._process_running = False 

def runpytest_subprocess(self, *args, timeout=None) -> RunResult:
        """Run pytest as a subprocess with given arguments.

        Any plugins added to the :py:attr:`plugins` list will be added using the
        ``-p`` command line option.  Additionally ``--basetemp`` is used to put
        any temporary files and directories in a numbered directory prefixed
        with "runpytest-" to not conflict with the normal numbered pytest
        location for temporary files and directories.

        :param args: the sequence of arguments to pass to the pytest subprocess
        :param timeout: the period in seconds after which to timeout and raise
            :py:class:`Testdir.TimeoutExpired`

        Returns a :py:class:`RunResult`.
        """
        __tracebackhide__ = True
        p = make_numbered_dir(root=Path(str(self.tmpdir)), prefix="runpytest-")
        args = ("--basetemp=%s" % p,) + args
        plugins = [x for x in self.plugins if isinstance(x, str)]
        if plugins:
            args = ("-p", plugins[0]) + args
        args = self._getpytestargs() + args
        return self.run(*args, timeout=timeout) 

def connect_adb_wifi(addr) -> Device:
    """
    Run adb connect, and then call connect_usb(..)

    Args:
        addr: ip+port which can be used for "adb connect" argument

    Raises:
        ConnectError
    """
    assert isinstance(addr, six.string_types)

    subprocess.call([adbutils.adb_path(), "connect", addr])
    try:
        subprocess.call([adbutils.adb_path(), "-s", addr, "wait-for-device"],
                        timeout=2)
    except subprocess.TimeoutExpired:
        raise ConnectError("Fail execute", "adb connect " + addr)
    return connect_usb(addr) 

def api_raw_host_ping(host_id: str or int) -> wrappers.Response:
    """
    Ping host by id
    :param host_id: id of host in list of hosts
    :return: flask response
    """
    markers = StorageData.SEARCH_MARKERS or StorageData.MARKERS
    try:
        host_info = markers[int(host_id)]
        if ping(host_info.get("ip")):
            return jsonify({"status": "online"})
        return jsonify({"status": "offline"})
    except (TimeoutExpired, TimeoutError):
        return jsonify({"error": "timeout"})
    except:
        return jsonify({"error": "ping error"}) 

def exec_and_log_fail(args):
    try:
        pipes = subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
        std_out, std_err = pipes.communicate(timeout = 180)
        ret_code = pipes.returncode
    except subprocess.TimeoutExpired as e:
        pipes.terminate()
        print("Timeout!")
        return False

    if ret_code:
        print("** stdout:")
        print(std_out)
        print("** stderr:")
        print(std_err)
        return False
    return True 

def exec(self, detail, files):
        # To avoid calling system-wide installed binaries
        filename = './' + self.config.name

        _exec = Runner.exec_ if detail.f_stdin is None else Runner.open_stdin
        stdin = detail.f_stdin if detail.f_stdin is not None else detail.stdin

        try:
            print(detail.cmd)
            expected = _exec(self, self.t_bin, [filename] + detail.cmd, stdin)
            actual = _exec(self, self.t_recompiled, [filename] + detail.cmd, stdin)
            return self.compare(expected, actual, files)
        except subprocess.TimeoutExpired as e:
            return result_data.TIMEOUT, "Timeout"
        except FileNotFoundError as e:
            return result_data.ERROR, "File not found" 

def create_mutation_run_trial(
    genome: Genome, target_idx: LocIndex, mutation_op: Any, test_cmds: List[str], max_runtime: float
) -> MutantTrialResult:
    """Run a single mutation trial by creating a new mutated cache file, running the
    test commands, and then removing the mutated cache file.

    Args:
        genome: the genome to mutate
        target_idx: the mutation location
        mutation_op: the mutation operation
        test_cmds: the test commands to execute with the mutated code
        max_runtime: timeout for the trial

    Returns:
        The mutation trial result
    """
    LOGGER.debug("Running trial for %s", mutation_op)

    mutant = genome.mutate(target_idx, mutation_op, write_cache=True)

    try:
        mutant_trial = subprocess.run(
            test_cmds,
            capture_output=capture_output(LOGGER.getEffectiveLevel()),
            timeout=max_runtime,
        )
        return_code = mutant_trial.returncode

    except subprocess.TimeoutExpired:
        return_code = 3

    cache.remove_existing_cache_files(mutant.src_file)

    return MutantTrialResult(
        mutant=MutantReport(
            src_file=mutant.src_file, src_idx=mutant.src_idx, mutation=mutant.mutation
        ),
        return_code=return_code,
    ) 

def runAFL(args, ENVS):
    p = subprocess.Popen(args, env = ENVS)
    while True:
        try:
            p.wait(timeout=30 * 60) # 30 min
        except subprocess.TimeoutExpired:
            # check the status to decide whether to continue or terminate
            # args[9] is the outdir
            if num_of_hangs(args[9]) > 0 or num_of_crashes(args[9]) > 0:
                continue
            else:
                p.terminate() 

def subproc_call(cmd, timeout=None):
    """
    Execute a command with timeout, and return STDOUT and STDERR

    Args:
        cmd(str): the command to execute.
        timeout(float): timeout in seconds.

    Returns:
        output(bytes), retcode(int). If timeout, retcode is -1.
    """
    try:
        output = subprocess.check_output(
            cmd, stderr=subprocess.STDOUT,
            shell=True, timeout=timeout)
        return output, 0
    except subprocess.TimeoutExpired as e:
        logger.warn("Command '{}' timeout!".format(cmd))
        if e.output:
            logger.warn(e.output.decode('utf-8'))
            return e.output, -1
        else:
            return "", -1
    except subprocess.CalledProcessError as e:
        logger.warn("Command '{}' failed, return code={}".format(cmd, e.returncode))
        logger.warn(e.output.decode('utf-8'))
        return e.output, e.returncode
    except Exception:
        logger.warn("Command '{}' failed to run.".format(cmd))
        return "", -2 

def _wait(process, timeout):
        if sys.version_info >= (3, 3):
            # python 3 has timeout feature built-in
            try:
                process.communicate(timeout=timeout)
            except subprocess.TimeoutExpired:
                pass
        else:
            # on Python 2 we need to simulate it
            delay = 0.01
            while process.poll() is None and timeout > 0:
                time.sleep(delay)
                timeout -= delay
        return process.poll() 

def communicate_timeout(self, timeout=None):  # pylint: disable=no-self-use,unused-argument
        """pass this method as a replacement to themocked Popen.communicate method"""
        process = mock.Mock()
        process.pid = 0
        if timeout:
            raise TimeoutExpired(process, timeout)
        return [bytes(self.partial_response, 'utf-8')]