Python subprocess.STDOUT Examples

def manage():
    def call(*args, **kwargs):
        ignore_errors = kwargs.pop("ignore_errors", False)
        assert not kwargs
        cmd = [
            sys.executable,
            os.path.join(os.path.dirname(__file__), "testprj", "manage.py"),
        ] + list(args)
        try:
            return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            if not ignore_errors:
                raise
            return e.output

    return call 

def _get_terminal_size_tput(self):
        # get terminal width
        # src: http://stackoverflow.com/questions/263890/how-do-i-find-the-width-height-of-a-terminal-window
        try:
            cols = int(
                subprocess.check_output(
                    shlex.split("tput cols"), stderr=subprocess.STDOUT
                )
            )
            rows = int(
                subprocess.check_output(
                    shlex.split("tput lines"), stderr=subprocess.STDOUT
                )
            )

            return (cols, rows)
        except:
            pass 

def convert_image(inpath, outpath, size):
    """Convert an image file using ``sips``.

    Args:
        inpath (str): Path of source file.
        outpath (str): Path to destination file.
        size (int): Width and height of destination image in pixels.

    Raises:
        RuntimeError: Raised if ``sips`` exits with non-zero status.
    """
    cmd = [
        b'sips',
        b'-z', str(size), str(size),
        inpath,
        b'--out', outpath]
    # log().debug(cmd)
    with open(os.devnull, 'w') as pipe:
        retcode = subprocess.call(cmd, stdout=pipe, stderr=subprocess.STDOUT)

    if retcode != 0:
        raise RuntimeError('sips exited with %d' % retcode) 

def _run_command(cmd, timeout_secs=300):
    """ Runs a command with a specified timeout.

    Args:
        cmd : list of string
            The command with arguments to run.
        timeout_secs: integer
            The timeout in seconds

    Returns:
        Returns the process and the output as a pair.
    """
    proc = subprocess.Popen(
        cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)

    for i in range(timeout_secs):
        sleep(1)
        if proc.poll() is not None:
            (out, _) = proc.communicate()
            return proc, out.decode('utf-8')

    proc.kill()
    return proc, "Timeout of %s secs exceeded." % timeout_secs 

def convert_image(inpath, outpath, size):
    """Convert an image file using ``sips``.

    Args:
        inpath (str): Path of source file.
        outpath (str): Path to destination file.
        size (int): Width and height of destination image in pixels.

    Raises:
        RuntimeError: Raised if ``sips`` exits with non-zero status.
    """
    cmd = [
        b'sips',
        b'-z', str(size), str(size),
        inpath,
        b'--out', outpath]
    # log().debug(cmd)
    with open(os.devnull, 'w') as pipe:
        retcode = subprocess.call(cmd, stdout=pipe, stderr=subprocess.STDOUT)

    if retcode != 0:
        raise RuntimeError('sips exited with %d' % retcode) 

def startIPy(opts):
  global ipy
  global respond
  global disconnected
  global connected
  global km
  respond = opts["respond"]
  connected = opts["connected"]
  disconnected = opts["disconnected"]

  try:
    if os.environ.get('LT_IPYTHON_PATH'):
      cmd = os.environ.get('LT_IPYTHON_PATH')
    elif os.path.isfile('bin/ipython'):
      cmd = 'bin/ipython'
    else:
      cmd = 'ipython'
    ipy = subprocess.Popen([cmd, 'kernel', '--pylab=inline'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=os.environ)
    #Start a thread listening to stdout
    t = threading.Thread(target=listenIPy)
    t.start()
    return True
  except:
    disconnected()
    return None 

def _custodia_cli(self, *args):
        env = os.environ.copy()
        env['PYTHONWARNINGS'] = 'ignore'
        pexec = shlex.split(env.get('CUSTODIAPYTHON', sys.executable))
        cli = pexec + [
            '-m', 'custodia.cli',
            '--verbose'
        ]
        cli.extend(args)

        try:
            # Python 2.7 doesn't have CalledProcessError.stderr
            output = subprocess.check_output(
                cli, env=env, stderr=subprocess.STDOUT
            )
        except subprocess.CalledProcessError as e:
            output = e.output
            if not isinstance(e.output, six.text_type):
                e.output = e.output.decode('utf-8')
            raise
        else:
            if not isinstance(output, six.text_type):
                output = output.decode('utf-8')
            return output 

def sign_apk(filename, keystore, storepass):
    """
    Use jarsigner to sign an APK file.

    :param filename: APK file on disk to sign (path)
    :param keystore: path to keystore
    :param storepass: your keystorage passphrase
    """
    from subprocess import Popen, PIPE, STDOUT
    # TODO use apksigner instead of jarsigner
    cmd = Popen([androconf.CONF["BIN_JARSIGNER"], "-sigalg", "MD5withRSA",
                 "-digestalg", "SHA1", "-storepass", storepass, "-keystore",
                 keystore, filename, "alias_name"],
                stdout=PIPE,
                stderr=STDOUT)
    stdout, stderr = cmd.communicate() 

def run(command, *args):
    """Helper function to run a subprocess in a Python lambda expression.

    :type command: string
    :param command: external command to run
    :type args: strings
    :param args: arguments for the external command
    :rtype: string
    :return: result of running the command
    """

    whole = [command]
    for arg in args:
        g = glob.glob(arg)
        if len(g) > 0:
            whole.extend(g)
        else:
            whole.append(arg)
    proc = subprocess.Popen(whole, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    returnCode = proc.wait()
    output = proc.stdout.read()
    if returnCode != 0:
        output += "\nsubprocesses failed with exit code {0}".format(returnCode)
    return output 

def get_git_revision():
    """When mdp is run from inside a git repository, this function
    returns the current revision that git-describe gives us.

    If mdp is installed (or git fails for some other reason),
    an empty string is returned.
    """
    # TODO: Introduce some fallback method that takes the info from a file
    revision = ''
    try:
        # we need to be sure that we call from the mdp dir
        mdp_dir = os.path.dirname(mdp.__file__)
        # --tags ensures that most revisions have a name even without
        # annotated tags
        # --dirty=+ appends a plus if the working copy is modified
        command = ["git", "describe", "--tags", "--dirty=+"]
        proc = Popen(command, stdout=PIPE, stderr=STDOUT, cwd=mdp_dir,
                     universal_newlines=True)
        exit_status = proc.wait()
        # only get the revision if command succeded
        if exit_status == 0:
            revision = proc.stdout.read().strip()
    except OSError:
        pass
    return revision 

def __enter__(self):
        logging.warning(f'Running s_server: "{self._command_line}"')
        args = shlex.split(self._command_line)
        try:
            self._process = subprocess.Popen(
                args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
            )
            self._server_io_manager = _OpenSslServerIOManager(self._process.stdout, self._process.stdin)

            # Block until s_server is ready to accept requests
            while not self._server_io_manager.is_server_ready:
                time.sleep(1)
                if self._process.poll() is not None:
                    # s_server has terminated early
                    raise RuntimeError("Could not start s_server")

        except Exception as e:
            logging.warning(f"Error while starting s_server: {e}")
            self._terminate_process()
            raise

        return self 

def test_simple(self):
    deps = self.FakeRecipeDeps()
    with deps.main_repo.write_recipe('foo') as recipe:
      recipe.DEPS = ['recipe_engine/python']
      recipe.RunSteps.write('''
        api.python.succeeding_step('hey there', "This is some narwhals.")
      ''')

    deps.main_repo.commit('save recipe')

    dest = self.tempdir()
    output, retcode = deps.main_repo.recipes_py('bundle', '--destination', dest)
    self.assertEqual(retcode, 0, 'bundling failed!\noutput:\n'+output)

    bat = '.bat' if sys.platform.startswith('win') else ''
    proc = subprocess.Popen(
        [os.path.join(dest, 'recipes'+bat), 'run', 'foo'],
        stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    output, _ = proc.communicate()
    self.assertEqual(proc.returncode, 0, 'running failed!\noutput:\n'+output)
    self.assertIn('narwhals', output) 

def _ninja_deps(cls):  # pylint: disable=no-self-argument
        deps = []

        try:
            subprocess.check_output(['ninja', '--version'], stderr=subprocess.STDOUT)
        except OSError:
            dep = mx.library('NINJA', False)
            if dep:
                deps.append(dep.qualifiedName())
                Ninja.binary = mx.join(dep.get_path(False), 'ninja')
            else:
                # necessary until GR-13214 is resolved
                mx.warn('Make `ninja` binary available via PATH to build native projects.')

        try:
            import ninja_syntax  # pylint: disable=unused-variable, unused-import
        except ImportError:
            dep = mx.library('NINJA_SYNTAX')
            deps.append(dep.qualifiedName())
            module_path = mx.join(dep.get_path(False), 'ninja_syntax-{}'.format(dep.version))
            mx.ensure_dir_exists(module_path)  # otherwise, import machinery will ignore it
            sys.path.append(module_path)

        return deps 

def stop_old_processes(hosts_file, prog_name):
    stop_args = ['python', '../../tools/kill-mxnet.py', hosts_file, 'python', prog_name]
    stop_args_str = ' '.join(stop_args)
    LOGGER.info('killing old remote processes\n %s', stop_args_str)
    stop = subprocess.check_output(stop_args, stderr=subprocess.STDOUT)
    LOGGER.debug(stop)
    time.sleep(1) 

def popen(fullcmd):
        p = subprocess.Popen(fullcmd, shell=True,
                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                             close_fds=True)
        return p.stdout 

def _run(self, *args, **kwargs):
        cmd = [self.binary, '-j', self.parallelism]
        if mx.get_opts().very_verbose:
            cmd += ['-v']
        cmd += args

        out = kwargs.get('out', mx.OutputCapture())
        err = kwargs.get('err', subprocess.STDOUT)
        if mx.get_opts().verbose:
            if callable(out) and '-n' not in args:
                out = mx.TeeOutputCapture(out)
            if callable(err):
                err = mx.TeeOutputCapture(err)

        try:
            rc = mx.run(cmd, nonZeroIsFatal=False, out=out, err=err, cwd=self.build_dir)
        except OSError as e:
            if e.errno != errno.EACCES:
                mx.abort('Error executing \'{}\': {}'.format(' '.join(cmd), str(e)))
            mx.logv('{} is not executable. Trying to change permissions...'.format(self.binary))
            os.chmod(self.binary, 0o755)
            self._run(*args, **kwargs)  # retry
        else:
            not rc or mx.abort(rc if mx.get_opts().verbose else out.data)  # pylint: disable=expression-not-assigned 

def meta(entries = FILES):
    '''
    metadata generator

    * takes a list of filenames and returns a 2d list:
      [0] absolute path
      [1] mtime
      [2] wc -w
      [3] timestamp "DD month YYYY at HH:MM"
      [4] entry date YYYY-MM-DD
      [5] author

    * sorted in reverse date order by [4]
    '''

    meta = []

    for filename in entries:
      mtime = os.path.getmtime(filename)
      try:
        wc = int(subprocess.check_output(["wc","-w",filename], stderr=subprocess.STDOUT).split()[0])
      except subprocess.CalledProcessError:
        wc = "???"
      timestamp = time.strftime("%Y-%m-%d at %H:%M", time.localtime(mtime))
      date = "-".join(util.parse_date(filename))
      author = os.path.split(os.path.split(os.path.split(os.path.split(filename)[0])[0])[0])[1]

      meta.append([filename, mtime, wc, timestamp, date, author])

    #meta.sort(key = lambda filename:filename[4])
    #meta.reverse()

    return meta 

def test_subprocess(self):
        if IOLoop.configured_class().__name__.endswith('LayeredTwistedIOLoop'):
            # This test fails non-deterministically with LayeredTwistedIOLoop.
            # (the read_until('\n') returns '\n' instead of 'hello\n')
            # This probably indicates a problem with either TornadoReactor
            # or TwistedIOLoop, but I haven't been able to track it down
            # and for now this is just causing spurious travis-ci failures.
            raise unittest.SkipTest("Subprocess tests not compatible with "
                                    "LayeredTwistedIOLoop")
        subproc = Subprocess([sys.executable, '-u', '-i'],
                             stdin=Subprocess.STREAM,
                             stdout=Subprocess.STREAM, stderr=subprocess.STDOUT,
                             io_loop=self.io_loop)
        self.addCleanup(lambda: os.kill(subproc.pid, signal.SIGTERM))
        subproc.stdout.read_until(b'>>> ', self.stop)
        self.wait()
        subproc.stdin.write(b"print('hello')\n")
        subproc.stdout.read_until(b'\n', self.stop)
        data = self.wait()
        self.assertEqual(data, b"hello\n")

        subproc.stdout.read_until(b">>> ", self.stop)
        self.wait()
        subproc.stdin.write(b"raise SystemExit\n")
        subproc.stdout.read_until_close(self.stop)
        data = self.wait()
        self.assertEqual(data, b"") 

def logs_present(self, statement, args=None):
        # Each test may manipulate and/or parse the options and then logs
        # a line at the 'info' level.  This level is ignored in the
        # logging module by default, but Tornado turns it on by default
        # so it is the easiest way to tell whether tornado's logging hooks
        # ran.
        IMPORT = 'from tornado.options import options, parse_command_line'
        LOG_INFO = 'import logging; logging.info("hello")'
        program = ';'.join([IMPORT, statement, LOG_INFO])
        proc = subprocess.Popen(
            [sys.executable, '-c', program] + (args or []),
            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        stdout, stderr = proc.communicate()
        self.assertEqual(proc.returncode, 0, 'process failed: %r' % stdout)
        return b'hello' in stdout 

def recipes_py(self, *args, **kwargs):
    """Runs `recipes.py` in this repo with the given args, just like a user
    might run it.

    Args:
      * args (List[str]) - the arguments to pass to recipes.py.

    Kwargs:
      * env (Dict[str, str]) - Extra environment variables to set while invoking
        recipes.py.

    Returns (output, retcode) where 'output' is the combined stdout/stderr from
    the command and retcode it's return code.
    """
    env = os.environ.copy()
    env.update(kwargs.pop('env', {}))
    if not any(r.has_protos for r in self.fake_recipe_deps.repos.itervalues()):
      pb_pkg_path = os.path.join(ROOT_DIR, '.recipe_deps', '_pb')
      args = ('--proto-override', pb_pkg_path) + args
    proc = subprocess.Popen(
        ('python', 'recipes.py')+args,
        cwd=self.path,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        env=env,
    )
    output, _ = proc.communicate()
    return output, proc.returncode 

def EncoderVersion(self):
    # The vpxenc command line tool outputs the version number of the
    # encoder as part of its error message on illegal arguments.
    try:
      subprocess.check_output([encoder.Tool('vpxenc')],
                              stderr=subprocess.STDOUT)
    except Exception, err:
      version_output = str(err.output)
      for line in version_output.split('\n'):
        match = re.match(r'\s+vp8\s+- (.+)$', line)
        if match:
          return match.group(1)
      raise encoder.Error('Did not find vp8 version string') 

def run_command(self):
        def do_run(*args):
            args = ['python3', 'vxapi.py'] + list(args)
            p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            (output, _) = p.communicate()
            output = output.decode('utf-8')
            self.output = output
            self.code = p.returncode
            print(output)

            return [p.returncode, output]

        return do_run 

def check_subprocess_output(command, message=None, stderr=subprocess.STDOUT, **proc_kwargs):
    if message:
        print(message)
    print("Executing: {0}".format(" ".join(command)))

    return subprocess.check_output(
        command,
        stderr=subprocess.STDOUT,
        **proc_kwargs
    ) 

def _custoda_cli(self, *extra_args, **kwargs):
        env = os.environ.copy()
        env['PYTHONWARNINGS'] = 'ignore'
        args = list(self.custodia_cli_args)
        args.extend(extra_args)
        try:
            out = subprocess.check_output(
                args, env=env, stderr=subprocess.STDOUT,
            )
        except subprocess.CalledProcessError as e:
            if e.returncode == 1:
                # HTTP error, reraise
                raise
            else:
                # other other
                out = e.output
                if not isinstance(out, six.text_type):
                    out = out.decode('utf-8')
                self.fail(out)

        if not isinstance(out, six.text_type):
            out = out.decode('utf-8')
        # remove trailing new line
        out = out.rstrip()
        if kwargs.get('split'):
            out = out.split('\n')
        return out 

def run(self, cmd, stdin=None, stderr=STDOUT, workdir=None):
        p = run(cmd, check=True, encoding="utf8", cwd=workdir,
                input=stdin, stdout=PIPE, stderr=stderr)

        return p.stdout 

def run_error(self, cmd, returncode, stdin=None, stderr=STDOUT, workdir=None):
        try:
            output = self.run(cmd, stdin=stdin, stderr=stderr, workdir=workdir)
        except CalledProcessError as e:
            if e.returncode != returncode:
                raise ValueError("Expected exit code {} but saw {}".format(returncode, e.returncode))
            else:
                output = e.stdout

        return output 

def kill_slaves(slave_kill_filename):
    """Kill all remote slaves which are stored in the given file.

    This functions is only meant for emergency situations, when something
    went wrong and the slaves have to be killed manually.
    """
    with open(slave_kill_filename) as tempfile:
        for line in tempfile:
            address, pid, ssh_pid = line.split(":")
            pid = int(pid)
            ssh_pid = int(ssh_pid)
            # open ssh connection to to kill remote slave
            proc = subprocess.Popen(["ssh","-T", address],
                                    stdin=subprocess.PIPE,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.STDOUT)
            proc.stdin.write("kill %d\n" % pid)
            proc.stdin.flush()
            # kill old ssh connection
            try:
                os.kill(ssh_pid, signal.SIGKILL)
            except:
                pass
            # a kill might prevent the kill command transmission
            # os.kill(proc.pid, signal.SIGQUIT)
            print "killed slave " + address + " (pid %d)" % pid
        print "all slaves killed." 

def run_cmd(cmd):
    '''Run shell command'''
    print(f'+ {cmd}')
    proc = subprocess.Popen(cmd, cwd=ROOT_DIR, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
    return proc 

def run(self, command):
    env = os.environ.copy()
    env["GOOEY"] = "1"
    self._process = subprocess.Popen(
      command.encode(sys.getfilesystemencoding()),
      bufsize=1, stdout=subprocess.PIPE,
      stderr=subprocess.STDOUT, shell=True, env=env)
    Pool(1).apply_async(self._forward_stdout, (self._process,)) 

def main():
    try:
        # receive sys_paths via stdin to be used in the wrapper
        python_executable = sys.stdin.readline()[:-1] # remove newline character
        sys_paths = []
        while True:
            sys_path = sys.stdin.readline()[:-1] # remove newline character
            if sys_path == "_done_":
                break
            sys_paths.append(sys_path)
        # assemble the command line for the wrapper by forwarding the arguments and
        cmd = ("nice %s %s pp_slave_wrapper.py" %
               (sys.argv[1], python_executable))
        for arg in sys.argv[2:]:
            cmd += " " + arg
        for sys_path in sys_paths:
            cmd += " " + sys_path
        # start the subprocess in which the slave process runs
        proc = subprocess.Popen(cmd, shell=True,
                                stdin=subprocess.PIPE,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)
        # print status message from slave process
        sys.stdout.write(proc.stdout.readline())
        sys.stdout.flush()
        # return the pid via stdout
        print proc.pid
        sys.stdout.flush()
    except Exception, e:
        print "Error while starting the server process."
        print e
        print -1
        sys.stdout.flush()