Python subprocess.html() Examples

def command_to_args(command_str):
        """
        convert a string bash command to an arguments list, to use with subprocess

        Most autoxtrabackup code creates a string command, e.g. "xtrabackup --prepare --target-dir..."
        If we run a string command with subprocess.Popen, we require shell=True.
        shell=True has security considerations (below), and we run autoxtrabackup with privileges (!).
        https://docs.python.org/3/library/subprocess.html#security-considerations
        So, convert to an args list and call Popen without shell=True.

        :param command_str: string command to execute as a subprocess
        :type command_str: str
        :return: list of args to pass to subprocess.Popen.
        :rtype: list
        """
        if isinstance(command_str, list):
            # already a list
            args = command_str
        elif isinstance(command_str, str):
            args = shlex.split(command_str)
        else:
            raise TypeError
        logger.debug("subprocess args are: {}".format(args))
        return args 

def setup_jedihttp(args=[]):
    def wrapper():
        hmac_file = hmaclib.temporary_hmac_secret_file(SECRET)
        command = [utils.python(),
                   PATH_TO_JEDIHTTP,
                   '--port', str(PORT),
                   '--log', 'debug',
                   '--hmac-file-secret', hmac_file] + args
        # Define environment variable to enable subprocesses coverage. See:
        # http://coverage.readthedocs.io/en/latest/subprocess.html
        env = os.environ.copy()
        env['COVERAGE_PROCESS_START'] = '.coveragerc'
        jedihttp = utils.safe_popen(command,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.STDOUT,
                                    env=env)
        wait_until_jedihttp_ready()
        return jedihttp
    return wrapper 

def command(self, cmd: list, **kwargs) -> bytes:
        """ Run a command at the tunnel target

        Args:
            cmd: list of strings that will be sent as a command to the target
            **kwargs: any keywork args that can be passed into
                subprocess.check_output. For more information, see:
                https://docs.python.org/3/library/subprocess.html#subprocess.check_output
        """
        run_cmd = ['ssh', '-p', str(self.port)] + self.opt_list + [self.target] + cmd
        log.debug('Running socket cmd: ' + ' '.join(run_cmd))
        if 'stdout' in kwargs:
            return subprocess.run(run_cmd, **kwargs, check=True, env={"PATH": os.environ["PATH"]})
        else:
            return subprocess.run(run_cmd, **kwargs, check=True, env={"PATH": os.environ["PATH"]},
                                  stdout=subprocess.PIPE).stdout 

def command(self, cmd: list, **kwargs) -> bytes:
        """ Run a command at the tunnel target

        Args:
            cmd: list of strings that will be sent as a command to the target
            **kwargs: any keywork args that can be passed into
                subprocess.check_output. For more information, see:
                https://docs.python.org/3/library/subprocess.html#subprocess.check_output
        """
        run_cmd = ['ssh', '-p', str(self.port)] + self.opt_list + [self.target] + cmd
        log.debug('Running socket cmd: ' + ' '.join(run_cmd))
        if 'stdout' in kwargs:
            return subprocess.run(run_cmd, **kwargs, check=True, env={"PATH": os.environ["PATH"]})
        else:
            return subprocess.run(run_cmd, **kwargs, check=True, env={"PATH": os.environ["PATH"]},
                                  stdout=subprocess.PIPE).stdout 

def stop(self):
        """ Stop this VM
        """
        self.running = False

        try:
            self.p.terminate()
        except ProcessLookupError:
            return

        try:
            self.p.communicate(timeout=10)
        except:
            try:
                # this construct is included as an example at
                # https://docs.python.org/3.6/library/subprocess.html but has
                # failed on me so wrapping in another try block. It was this
                # communicate() that failed with:
                # ValueError: Invalid file object: <_io.TextIOWrapper name=3 encoding='ANSI_X3.4-1968'>
                self.p.kill()
                self.p.communicate(timeout=10)
            except:
                # just assume it's dead or will die?
                self.p.wait(timeout=10) 

def wait_for_standing_subprocess(proc, timeout=None):
    """Waits for a subprocess started by start_standing_subprocess to finish
    or times out.

    Propagates the exception raised by the subprocess.wait(.) function.
    The subprocess.TimeoutExpired exception is raised if the process timed-out
    rather than terminating.

    If no exception is raised: the subprocess terminated on its own. No need
    to call stop_standing_subprocess() to kill it.

    If an exception is raised: the subprocess is still alive - it did not
    terminate. Either call stop_standing_subprocess() to kill it, or call
    wait_for_standing_subprocess() to keep waiting for it to terminate on its
    own.

    If the corresponding subprocess command generates a large amount of output
    and this method is called with a timeout value, then the command can hang
    indefinitely. See http://go/pylib/subprocess.html#subprocess.Popen.wait

    This function does not support Python 2.

    Args:
        p: Subprocess to wait for.
        timeout: An integer number of seconds to wait before timing out.
    """
    proc.wait(timeout) 

def _replace_module():
    """Dirty hack to replace the module object in order to access
    deprecated module constants, see:
    http://www.dr-josiah.com/2013/12/properties-on-python-modules.html
    """
    class ModuleWrapper(object):

        def __repr__(self):
            return repr(self._module)
        __str__ = __repr__

        @property
        def NUM_CPUS(self):
            msg = "NUM_CPUS constant is deprecated; use cpu_count() instead"
            warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
            return cpu_count()

        @property
        def BOOT_TIME(self):
            msg = "BOOT_TIME constant is deprecated; use boot_time() instead"
            warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
            return boot_time()

        @property
        def TOTAL_PHYMEM(self):
            msg = "TOTAL_PHYMEM constant is deprecated; " \
                  "use virtual_memory().total instead"
            warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
            return virtual_memory().total

    mod = ModuleWrapper()
    mod.__dict__ = globals()
    mod._module = sys.modules[__name__]
    sys.modules[__name__] = mod 

def _replace_module():
    """Dirty hack to replace the module object in order to access
    deprecated module constants, see:
    http://www.dr-josiah.com/2013/12/properties-on-python-modules.html
    """
    class ModuleWrapper(object):

        def __repr__(self):
            return repr(self._module)
        __str__ = __repr__

        @property
        def NUM_CPUS(self):
            msg = "NUM_CPUS constant is deprecated; use cpu_count() instead"
            warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
            return cpu_count()

        @property
        def BOOT_TIME(self):
            msg = "BOOT_TIME constant is deprecated; use boot_time() instead"
            warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
            return boot_time()

        @property
        def TOTAL_PHYMEM(self):
            msg = "TOTAL_PHYMEM constant is deprecated; " \
                  "use virtual_memory().total instead"
            warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
            return virtual_memory().total

    mod = ModuleWrapper()
    mod.__dict__ = globals()
    mod._module = sys.modules[__name__]
    sys.modules[__name__] = mod 

def ExecLinkWrapper(self, arch, use_separate_mspdbsrv, *args):
    """Filter diagnostic output from link that looks like:
    '   Creating library ui.dll.lib and object ui.dll.exp'
    This happens when there are exports from the dll or exe.
    """
    env = self._GetEnv(arch)
    if use_separate_mspdbsrv == 'True':
      self._UseSeparateMspdbsrv(env, args)
    if sys.platform == 'win32':
      args = list(args)  # *args is a tuple by default, which is read-only.
      args[0] = args[0].replace('/', '\\')
    # https://docs.python.org/2/library/subprocess.html:
    # "On Unix with shell=True [...] if args is a sequence, the first item
    # specifies the command string, and any additional items will be treated as
    # additional arguments to the shell itself.  That is to say, Popen does the
    # equivalent of:
    #   Popen(['/bin/sh', '-c', args[0], args[1], ...])"
    # For that reason, since going through the shell doesn't seem necessary on
    # non-Windows don't do that there.
    link = subprocess.Popen(args, shell=sys.platform == 'win32', env=env,
                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    out, _ = link.communicate()
    for line in out.splitlines():
      if (not line.startswith('   Creating library ') and
          not line.startswith('Generating code') and
          not line.startswith('Finished generating code')):
        print line
    return link.returncode 

def ExecLinkWrapper(self, arch, use_separate_mspdbsrv, *args):
    """Filter diagnostic output from link that looks like:
    '   Creating library ui.dll.lib and object ui.dll.exp'
    This happens when there are exports from the dll or exe.
    """
    env = self._GetEnv(arch)
    if use_separate_mspdbsrv == 'True':
      self._UseSeparateMspdbsrv(env, args)
    if sys.platform == 'win32':
      args = list(args)  # *args is a tuple by default, which is read-only.
      args[0] = args[0].replace('/', '\\')
    # https://docs.python.org/2/library/subprocess.html:
    # "On Unix with shell=True [...] if args is a sequence, the first item
    # specifies the command string, and any additional items will be treated as
    # additional arguments to the shell itself.  That is to say, Popen does the
    # equivalent of:
    #   Popen(['/bin/sh', '-c', args[0], args[1], ...])"
    # For that reason, since going through the shell doesn't seem necessary on
    # non-Windows don't do that there.
    link = subprocess.Popen(args, shell=sys.platform == 'win32', env=env,
                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    out, _ = link.communicate()
    for line in out.splitlines():
      if (not line.startswith('   Creating library ') and
          not line.startswith('Generating code') and
          not line.startswith('Finished generating code')):
        print line
    return link.returncode 

def _RunCommand(self, command, stdout=None, stderr=None):
    """Runs a command.

    Args:
      command (list[str]): full command to run, as expected by the Popen()
        constructor (see the documentation:
        https://docs.python.org/2/library/subprocess.html#popen-constructor)
      stdout (Optional[str]): path to file to send stdout to.
      stderr (Optional[str]): path to file to send stderr to.

    Returns:
      bool: True if the command ran successfully.
    """
    if command[0].endswith('py'):
      command.insert(0, sys.executable)
    logging.info('Running: {0:s}'.format(' '.join(command)))
    child = subprocess.Popen(command, stdout=stdout, stderr=stderr)
    child.communicate()
    exit_code = child.returncode

    if exit_code != 0:
      logging.error('Running: "{0:s}" failed (exit code {1:d}).'.format(
          command, exit_code))
      return False

    return True 

def test_deployment():
    # https://docs.python.org/3/library/subprocess.html#subprocess.run
    # using check=True will throw an exception if a non-zero exit code is returned, saving us the need to assert
    # using timeout=10 will throw an exception if the process doesn't return within 10 seconds
    # Enables the deployment
    process_result = subprocess.run('kubectl apply -f ../deploy/', check=True, shell=True, timeout=10) 

def kubectl_proxy():
    # establish proxy for kubectl communications
    # https://docs.python.org/3/library/subprocess.html#subprocess-replacements
    proxy = subprocess.Popen("kubectl proxy &", stdout=subprocess.PIPE, shell=True)
    yield
    # terminate the proxy
    proxy.kill() 

def edit():
    """
    Manually edit the bibfile database in text editor.

    Resources
    ---------
    https://stackoverflow.com/questions/17317219/
    https://docs.python.org/3.6/library/subprocess.html
    """
    export(load(), u.BM_TMP_BIB(), meta=True)
    # Open database.bib into temporary file with default text editor
    if sys.platform == "win32":
        os.startfile(u.BM_TMP_BIB())
    else:
        opener = cm.get('text_editor')
        if opener == 'default':
            opener = "open" if sys.platform == "darwin" else "xdg-open"
        subprocess.call([opener, u.BM_TMP_BIB()])
    # Launch input() call to wait for user to save edits:
    dummy = input("Press ENTER to continue after you edit, save, and close "
                  "the bib file.")
    # Check edits:
    try:
        new = loadfile(u.BM_TMP_BIB())
    finally:
        # Always delete the tmp file:
        os.remove(u.BM_TMP_BIB())
    # Update database if everything went fine:
    with u.ignored(OSError):
        os.remove(u.BM_DATABASE())
    merge(new=new) 

def run_command(self, command):
        """
        Run command in subprocess.

        Example from:
            http://asyncio.readthedocs.io/en/latest/subprocess.html
        """
        # Create subprocess
        process = await asyncio.create_subprocess_shell(
            command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
        )

        # Status
        print("Started: '%s', pid: '%s'" % (command, process.pid), flush=True)

        # Wait for the subprocess to finish
        stdout, stderr = await process.communicate()

        # Progress
        if process.returncode == 0:
            print(
                "Done: %s, pid=%s, result: %s"
                % (command, process.pid, stdout.decode().strip()),
                flush=True,
            )
        else:
            print(
                "Failed: %s, pid=%s, result: %s"
                % (command, process.pid, stderr.decode().strip()),
                flush=True,
            )

        # Result
        result = stdout.decode().strip()

        # Return stdout and return code
        return result, process.returncode 

def ExecLinkWrapper(self, arch, use_separate_mspdbsrv, *args):
    """Filter diagnostic output from link that looks like:
    '   Creating library ui.dll.lib and object ui.dll.exp'
    This happens when there are exports from the dll or exe.
    """
    env = self._GetEnv(arch)
    if use_separate_mspdbsrv == 'True':
      self._UseSeparateMspdbsrv(env, args)
    if sys.platform == 'win32':
      args = list(args)  # *args is a tuple by default, which is read-only.
      args[0] = args[0].replace('/', '\\')
    # https://docs.python.org/2/library/subprocess.html:
    # "On Unix with shell=True [...] if args is a sequence, the first item
    # specifies the command string, and any additional items will be treated as
    # additional arguments to the shell itself.  That is to say, Popen does the
    # equivalent of:
    #   Popen(['/bin/sh', '-c', args[0], args[1], ...])"
    # For that reason, since going through the shell doesn't seem necessary on
    # non-Windows don't do that there.
    link = subprocess.Popen(args, shell=sys.platform == 'win32', env=env,
                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    out, _ = link.communicate()
    for line in out.splitlines():
      if (not line.startswith('   Creating library ') and
          not line.startswith('Generating code') and
          not line.startswith('Finished generating code')):
        print(line)
    return link.returncode 

def __init__(self, *args, **kwargs):
            # CREATE_NEW_PROCESS_GROUP is used to send Ctrl+C on Windows:
            # https://docs.python.org/3/library/subprocess.html#subprocess.Popen.send_signal
            new_pgroup = subprocess.CREATE_NEW_PROCESS_GROUP
            flags_to_add = 0
            if ray.utils.detect_fate_sharing_support():
                # If we don't have kernel-mode fate-sharing, then don't do this
                # because our children need to be in out process group for
                # the process reaper to properly terminate them.
                flags_to_add = new_pgroup
            flags_key = "creationflags"
            if flags_to_add:
                kwargs[flags_key] = (kwargs.get(flags_key) or 0) | flags_to_add
            self._use_signals = (kwargs[flags_key] & new_pgroup)
            super(ConsolePopen, self).__init__(*args, **kwargs) 

def ExecLinkWrapper(self, arch, use_separate_mspdbsrv, *args):
    """Filter diagnostic output from link that looks like:
    '   Creating library ui.dll.lib and object ui.dll.exp'
    This happens when there are exports from the dll or exe.
    """
    env = self._GetEnv(arch)
    if use_separate_mspdbsrv == 'True':
      self._UseSeparateMspdbsrv(env, args)
    if sys.platform == 'win32':
      args = list(args)  # *args is a tuple by default, which is read-only.
      args[0] = args[0].replace('/', '\\')
    # https://docs.python.org/2/library/subprocess.html:
    # "On Unix with shell=True [...] if args is a sequence, the first item
    # specifies the command string, and any additional items will be treated as
    # additional arguments to the shell itself.  That is to say, Popen does the
    # equivalent of:
    #   Popen(['/bin/sh', '-c', args[0], args[1], ...])"
    # For that reason, since going through the shell doesn't seem necessary on
    # non-Windows don't do that there.
    link = subprocess.Popen(args, shell=sys.platform == 'win32', env=env,
                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    out, _ = link.communicate()
    for line in out.splitlines():
      if (not line.startswith('   Creating library ') and
          not line.startswith('Generating code') and
          not line.startswith('Finished generating code')):
        print(line)
    return link.returncode 

def _open_subprocess(self, args=None, passphrase=False):
        """Open a pipe to a GPG subprocess and return the file objects for
        communicating with it.

        :param list args: A list of strings of options and flags to pass to
                          ``GPG.binary``. This is input safe, meaning that
                          these values go through strict checks (see
                          ``parsers._sanitise_list``) before being passed to to
                          the input file descriptor for the GnuPG process.
                          Each string should be given exactly as it would be on
                          the commandline interface to GnuPG,
                          e.g. ["--cipher-algo AES256", "--default-key
                          A3ADB67A2CDB8B35"].

        :param bool passphrase: If True, the passphrase will be sent to the
                                stdin file descriptor for the attached GnuPG
                                process.
        """
        ## see http://docs.python.org/2/library/subprocess.html#converting-an\
        ##    -argument-sequence-to-a-string-on-windows
        cmd = shlex.split(' '.join(self._make_args(args, passphrase)))
        log.debug("Sending command to GnuPG process:%s%s" % (os.linesep, cmd))

        if platform.system() == "Windows":
            # TODO figure out what the hell is going on there.
            expand_shell = True
        else:
            expand_shell = False

        return subprocess.Popen(cmd, shell=expand_shell, stdin=subprocess.PIPE,
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                env={'LANGUAGE': 'en'}) 

def launch_application(self, app, alias=None):
        '''Launches an application.

        Executes the string argument ``app`` as a separate process with
        Python's
        ``[https://docs.python.org/2/library/subprocess.html|subprocess]``
        module. It should therefore be the exact command you would use to
        launch the application from command line.

        On Windows, if you are using relative or absolute paths in ``app``,
        enclose the command with double quotes:

        | Launch Application | "C:\\my folder\\myprogram.exe" | # Needs quotes       |
        | Launch Application | myprogram.exe                  | # No need for quotes |

        Returns automatically generated alias which can be used with `Terminate
        Application`.

        Automatically generated alias can be overridden by providing ``alias``
        yourself.
        '''
        if not alias:
            alias = str(len(self.open_applications))
        process = subprocess.Popen(shlex.split(app))
        self.open_applications[alias] = process
        return alias 

def _RunCommand(self, command, stdout=None, stderr=None):
    """Runs a command.

    Args:
      command (list[str]): full command to run, as expected by the Popen()
        constructor (see the documentation:
        https://docs.python.org/2/library/subprocess.html#popen-constructor)
      stdout (Optional[str]): path to file to send stdout to.
      stderr (Optional[str]): path to file to send stderr to.

    Returns:
      bool: True if the command ran successfully.
    """
    if command[0].endswith('py'):
      command.insert(0, sys.executable)
    command_string = ' '.join(command)
    logging.info('Running: {0:s}'.format(command_string))
    child = subprocess.Popen(command, stdout=stdout, stderr=stderr)
    child.communicate()
    exit_code = child.returncode

    if exit_code != 0:
      logging.error('Running: "{0:s}" failed (exit code {1:d}).'.format(
          command_string, exit_code))
      return False

    return True

  # pylint: disable=redundant-returns-doc 

def __run_cmd(self):
        '''Run the command provided and return stdout and stderr.'''
        # In order to get coverage working for tests run through
        # subprocess, we need to set COVERAGE_PROCESS_START to point
        # to a .coveragerc, and trick the python interpreter into
        # calling coverage.process_startup() before doing anything
        # else.

        # This is documented here, though it doesn't say what
        # COVERAGE_PROCESS_START needs to contain:
        # http://coverage.readthedocs.io/en/coverage-4.2/subprocess.html

        # The way we're tricking python into running
        # coverage.process_startup() is by putting it in the .pth file
        # in site-packages. This may not work on all environments, for
        # example, if site-packages is read-only, but it should work
        # in the places we're using to test.

        # Also, due to https://github.com/pypa/virtualenv/issues/355,
        # we can't use site.getsitepackages(), but
        # distutils.sysconfig.get_python_lib() works.
        core_dir = os.path.dirname(os.path.dirname(__file__))
        if sys.gettrace():
            # Only add this .pth file if we're running under the
            # coverage tool, which is the most likely reason that
            # sys.gettrace() would be set to anything.
            site_packages = sysconfig.get_python_lib()
            with tempfile.NamedTemporaryFile(
                    mode='w', dir=site_packages, suffix='.pth',
                    delete=False) as pth:
                pth.write('import coverage; coverage.process_startup()\n')
                pth.flush()

            coverage_config = os.path.join(core_dir, '.coveragerc')
            env = {'COVERAGE_PROCESS_START': coverage_config, **os.environ}
            to_remove = pth.name
        else:
            env = None
            to_remove = None
        try:
            cwd = core_dir if self.cwd is None else self.cwd
            proc = subprocess.run(
                self.cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                env=env, cwd=cwd)
        finally:
            if to_remove:
                os.remove(to_remove)
        return proc.stdout.decode(), proc.stderr.decode()