Python subprocess.PIPE Examples

def call(self, cmd, **kwargs):
        print('Running "{}"'.format(cmd), file=sys.stderr)
        expect = kwargs.pop("expect", [dict(return_codes=[os.EX_OK], stdout=None, stderr=None)])
        process = subprocess.Popen(cmd, stdin=kwargs.get("stdin", subprocess.PIPE), stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE, **kwargs)
        out, err = process.communicate()
        return_code = process.poll()
        out = out.decode(sys.stdin.encoding)
        err = err.decode(sys.stdin.encoding)

        def match(return_code, out, err, expected):
            exit_ok = return_code in expected["return_codes"]
            stdout_ok = re.search(expected.get("stdout") or "", out)
            stderr_ok = re.search(expected.get("stderr") or "", err)
            return exit_ok and stdout_ok and stderr_ok
        if not any(match(return_code, out, err, exp) for exp in expect):
            print(err)
            e = subprocess.CalledProcessError(return_code, cmd, output=out)
            e.stdout, e.stderr = out, err
            raise e
        return self.SubprocessResult(out, err, return_code) 

def pipe_fopen(command, mode, background=True):
    if mode not in ["rb", "r"]:
        raise RuntimeError("Now only support input from pipe")

    p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)

    def background_command_waiter(command, p):
        p.wait()
        if p.returncode != 0:
            warnings.warn("Command \"{0}\" exited with status {1}".format(
                command, p.returncode))
            _thread.interrupt_main()

    if background:
        thread = threading.Thread(target=background_command_waiter,
                                  args=(command, p))
        # exits abnormally if main thread is terminated .
        thread.daemon = True
        thread.start()
    else:
        background_command_waiter(command, p)
    return p.stdout 

def alignProcWrapper(ref, seq):
    cmd = "python {} {} {}".format(
        os.path.realpath(__file__),
        ref,
        seq)
    
    proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = proc.communicate()

    if proc.returncode != 0:
        return None

    fields = out.split()
    strand = fields[0].decode()
    
    aln = Aln(int(fields[1]), int(fields[2]), fields[3].decode(), int(fields[4]), int(fields[5]))

    return strand, aln 

def update_version_py():
    if not os.path.isdir(".git"):
        print("This does not appear to be a Git repository.")
        return
    try:
        # p = subprocess.Popen(["git", "describe","--tags", "--always"],
        #        stdout=subprocess.PIPE)
        p = subprocess.Popen("git rev-list HEAD --count".split(),
                             stdout=subprocess.PIPE)

    except EnvironmentError:
        print("unable to run git, leaving eden/_version.py alone")
        return
    stdout = p.communicate()[0]
    if p.returncode != 0:
        print("unable to run git, leaving eden/_version.py alone")
        return
    ver = "0.3."+stdout.strip()
    # ver = str(int(ver,16)) # pypi doesnt like base 16
    f = open("eden/_version.py", "w")
    f.write(VERSION_PY % ver)
    f.close()
    print("set eden/_version.py to '%s'" % ver) 

def get_num_triggering_samples(signature, samples):
    """
        Get number of samples triggering ClamAV signature _signature_.
        :param signature: A dictionary with keys 'type' for the signature type
            and 'signature' for the signature string.
        :param samples: A list of sample paths to scan.
        :returns: The number of samples triggering this signature.
    """
    handle, temp_sig = tempfile.mkstemp(suffix = "." + signature["type"])
    try:
        with os.fdopen(handle, "w") as f:
            f.write(signature["signature"])
        proc_clamscan = subprocess.Popen(["clamscan", 
                                          "-d", temp_sig,
                                          "--no-summary", "--infected"] + samples, 
                                         stdout = subprocess.PIPE,
                                         stderr = subprocess.PIPE)
        stdout, stderr = proc_clamscan.communicate()
        if not stdout:
            return 0
        else:
            return len(stdout.strip().split("\n"))
    finally:
        os.unlink(temp_sig) 

def disas_objdump(b):
    with open("/dev/shm/shifter", "w") as f:
        f.write(b)
    if arch == "64":
        dis, errors = subprocess.Popen("objdump -D --insn-width=256 -b binary \
                -mi386 -Mx86-64 /dev/shm/shifter | head -8 | tail -1",
                stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True).communicate()
    else:
        dis, errors = subprocess.Popen("objdump -D --insn-width=256 -b binary \
                -mi386 /dev/shm/shifter | head -8 | tail -1",
                stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True).communicate()
    dis = dis[6:] # address
    raw = dis[:256*3].replace(" ","")
    dis = dis[256*3:].strip().split(None, 2)
    mnemonic = dis[0]
    if len(dis) > 1:
        op_str = dis[1]
    else:
        op_str = ""
    if mnemonic == "(bad)":
        mnemonic = "(unk)"
        insn = ""
        op_str = ""
    size = len(raw)/2
    return (mnemonic, op_str, size) 

def dd_iso_image(self, input_, output, gui_update, status_update):
        ''' Implementation for OS that use dd to write the iso image. 
        '''
        in_file_size = os.path.getsize(input_)
        cmd = [self.dd_exe, 'if=' + input_,
               'of=' + self.physical_disk(output), 'bs=1M']
        self.dd_iso_image_add_args(cmd, input_, output)
        kw_args = {
            'stdout' : subprocess.PIPE,
            'stderr' : subprocess.PIPE,
            'shell'  : False,
            }
        self.add_dd_iso_image_popen_args(kw_args)
        self.dd_iso_image_prepare(input, output, status_update)
        log('Executing => ' + str(cmd))
        dd_process = subprocess.Popen(cmd, **kw_args)
        output_q = queue.Queue()
        while dd_process.poll() is None:
            self.dd_iso_image_readoutput(dd_process, gui_update, in_file_size,
                                         output_q)
        output_lines = [output_q.get() for i in range(output_q.qsize())]
        for l in output_lines:
            log('dd: ' + l)
        return self.dd_iso_image_interpret_result(
            dd_process.returncode, output_lines) 

def get_perf(filename):
    ''' run conlleval.pl perl script to obtain
    precision/recall and F1 score '''
    _conlleval = PREFIX + 'conlleval'
    if not isfile(_conlleval):
        #download('http://www-etud.iro.umontreal.ca/~mesnilgr/atis/conlleval.pl') 
        os.system('wget https://www.comp.nus.edu.sg/%7Ekanmy/courses/practicalNLP_2008/packages/conlleval.pl')
        chmod('conlleval.pl', stat.S_IRWXU) # give the execute permissions
    
    out = []
    proc = subprocess.Popen(["perl", _conlleval], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    stdout, _ = proc.communicate(open(filename).read())
    for line in stdout.split('\n'):
        if 'accuracy' in line:
            out = line.split()
            break
    
    # out = ['accuracy:', '16.26%;', 'precision:', '0.00%;', 'recall:', '0.00%;', 'FB1:', '0.00']
    precision = float(out[3][:-2])
    recall    = float(out[5][:-2])
    f1score   = float(out[7])

    return {'p':precision, 'r':recall, 'f1':f1score} 

def get_git_hash():

    def _minimal_ext_cmd(cmd):
        # construct minimal environment
        env = {}
        for k in ['SYSTEMROOT', 'PATH', 'HOME']:
            v = os.environ.get(k)
            if v is not None:
                env[k] = v
        # LANGUAGE is used on win32
        env['LANGUAGE'] = 'C'
        env['LANG'] = 'C'
        env['LC_ALL'] = 'C'
        out = subprocess.Popen(
            cmd, stdout=subprocess.PIPE, env=env).communicate()[0]
        return out

    try:
        out = _minimal_ext_cmd(['git', 'rev-parse', 'HEAD'])
        sha = out.strip().decode('ascii')
    except OSError:
        sha = 'unknown'

    return sha 

def start(self):
        self.command = "%s %s -%c -R %s -s %d" % \
                (
                    INJECTOR,
                    " ".join(self.settings.args),
                    self.settings.synth_mode,
                    "-0" if self.settings.root else "",
                    self.settings.seed
                )
        self.process = subprocess.Popen(
            "exec %s" % self.command,
            shell=True,
            stdout=subprocess.PIPE,
            stdin=subprocess.PIPE,
            preexec_fn=os.setsid
            ) 

def gpt_device(self, dev_name):
        disk_dev = self.physical_disk(dev_name)
        cmd = ['parted', disk_dev, '-s', 'print']
        with open(os.devnull) as devnull:
            p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE, stdin=devnull)
            _cmd_out, _err_out = p.communicate()
            p.wait()
        if p.returncode != 0:
            lang = os.getenv('LANG')
            encoding = lang.rsplit('.')[-1] if lang else 'utf-8'
            raise RuntimeError(str(_err_out, encoding))
        subprocess.check_call(['partprobe', disk_dev])
        if b'msdos' in _cmd_out:
            return False
        if b'gpt' in _cmd_out:
            return True
        raise RuntimeError("Disk '%s' is uninitialized and not usable." %
                           disk_dev) 

def calculate(self):
        """
        Calculate the elastic constant given Potential class.
        """
        with ScratchDir('.'):
            input_file = self._setup()
            p = subprocess.Popen([self.LMP_EXE, '-in', input_file],
                                 stdout=subprocess.PIPE)
            stdout = p.communicate()[0]
            rc = p.returncode
            if rc != 0:
                error_msg = 'LAMMPS exited with return code %d' % rc
                msg = stdout.decode("utf-8").split('\n')[:-1]
                try:
                    error_line = [i for i, m in enumerate(msg)
                                  if m.startswith('ERROR')][0]
                    error_msg += ', '.join([e for e in msg[error_line:]])
                except Exception:
                    error_msg += msg[-1]
                raise RuntimeError(error_msg)
            result = self._parse()
        return result 

def calculate(self):
        """
        Calculate the NEB barrier given Potential class.
        """
        with ScratchDir('.'):
            input_file = self._setup()
            p = subprocess.Popen(['mpirun', '-n', str(self.num_replicas),
                                  'lmp_mpi', '-partition', '{}x1'.format(self.num_replicas),
                                  '-in', input_file],
                                 stdout=subprocess.PIPE)
            stdout = p.communicate()[0]
            rc = p.returncode
            if rc != 0:
                error_msg = 'LAMMPS exited with return code %d' % rc
                msg = stdout.decode("utf-8").split('\n')[:-1]
                try:
                    error_line = [i for i, m in enumerate(msg)
                                  if m.startswith('ERROR')][0]
                    error_msg += ', '.join([e for e in msg[error_line:]])
                except Exception:
                    error_msg += msg[-1]
                raise RuntimeError(error_msg)
            result = self._parse()
        return result 

def calculate(self):
        """
        Calculate the vacancy formation given Potential class.
        """
        with ScratchDir('.'):
            input_file, energy_per_atom, num_atoms = self._setup()
            p = subprocess.Popen([self.LMP_EXE, '-in', input_file], stdout=subprocess.PIPE)
            stdout = p.communicate()[0]

            rc = p.returncode
            if rc != 0:
                error_msg = 'LAMMPS exited with return code %d' % rc
                msg = stdout.decode("utf-8").split('\n')[:-1]
                try:
                    error_line = [i for i, m in enumerate(msg)
                                  if m.startswith('ERROR')][0]
                    error_msg += ', '.join([e for e in msg[error_line:]])
                except Exception:
                    error_msg += msg[-1]
                raise RuntimeError(error_msg)
            defect_energy, _, _ = self._parse()
        defect_formation_energy = defect_energy - energy_per_atom * num_atoms

        return defect_formation_energy 

def _run_command(cmd, timeout_secs=300):
    """ Runs a command with a specified timeout.

    Args:
        cmd : list of string
            The command with arguments to run.
        timeout_secs: integer
            The timeout in seconds

    Returns:
        Returns the process and the output as a pair.
    """
    proc = subprocess.Popen(
        cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)

    for i in range(timeout_secs):
        sleep(1)
        if proc.poll() is not None:
            (out, _) = proc.communicate()
            return proc, out.decode('utf-8')

    proc.kill()
    return proc, "Timeout of %s secs exceeded." % timeout_secs 

def communication_initialization(self):
		self.clients = []
		if self.serverorclient:
			if self.os_type == common.OS_LINUX:
				ps = subprocess.Popen(["cat", "/proc/sys/net/ipv4/icmp_echo_ignore_all"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
				(stdout, stderr) = ps.communicate()
				if stderr:
					common.internal_print("Error: deleting default route: {0}".format(stderr), -1)
					sys.exit(-1)
				self.orig_ieia_value = stdout[0:1]
				os.system("echo 1 > /proc/sys/net/ipv4/icmp_echo_ignore_all")

		if self.serverorclient:
			self.ICMP_send = self.icmp.ICMP_ECHO_RESPONSE
		else:
			self.ICMP_send = self.icmp.ICMP_ECHO_REQUEST
		return 

def lin_check_default_route(self):
		# get default gateway(s)
		ps = subprocess.Popen(["route", "-n"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		(stdout, stderr) = ps.communicate()

		if stderr != "":
			common.internal_print("Route error: {0}".format(stderr), -1)
			sys.exit(-1)

		lines = stdout.split("\n")
		default_route_number = 0
		for line in lines:
			if line[0:7] == "0.0.0.0":
				default_route_number += 1

		if default_route_number < 1:
			common.internal_print("No default route. Please set up your routing before executing the tool", -1)
			sys.exit(-1)
		if default_route_number > 1:
			common.internal_print("More than one default route. This should be reviewed before executing the tool.", -1)
			sys.exit(-1)

		return 

def lin_set_intermediate_route(self, serverip, proxyip):
		common.internal_print("Changing route table for intermediate hop")
		ps = subprocess.Popen(["route", "delete", serverip, "gw", self.orig_default_gw], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		(stdout, stderr) = ps.communicate()
		if stderr:
			common.internal_print("Error: delete old route: {0}".format(stderr), -1)
			sys.exit(-1)

		ps = subprocess.Popen(["route", "add", "-host", proxyip, "gw", self.orig_default_gw], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		(stdout, stderr) = ps.communicate()
		if stderr:
			if not "File exists" in stderr:
				common.internal_print("Error: adding server route: {0}".format(stderr), -1)
				sys.exit(-1)

		return 

def mac_set_ip_address(self, dev, ip, serverip, netmask):
		ifr = struct.pack('<16sBBHIIIBBHIIIBBHIII',
			self.iface_name,
			16, socket.AF_INET, 0, struct.unpack('<L', socket.inet_pton(socket.AF_INET, ip))[0], 0, 0,
			16, socket.AF_INET, 0, struct.unpack('<L', socket.inet_pton(socket.AF_INET, serverip))[0], 0, 0,
			16, 0, 0, struct.unpack('<L', socket.inet_pton(socket.AF_INET, "255.255.255.255"))[0], 0, 0)
		try:
			sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
			fcntl.ioctl(sock, self.IOCTL_MACOSX_SIOCAIFADDR, ifr)
		except Exception as e:
			common.internal_print("Something went wrong with setting up the interface.", -1)
			print(e)
			sys.exit(-1)

		# adding new route for forwarding packets properly.
		integer_ip = struct.unpack(">I", socket.inet_pton(socket.AF_INET, serverip))[0]
		rangeip = socket.inet_ntop(socket.AF_INET, struct.pack(">I", integer_ip & ((2**int(netmask))-1)<<32-int(netmask)))
		ps = subprocess.Popen(["route", "add", "-net", rangeip+"/"+netmask, serverip], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		(stdout, stderr) = ps.communicate()
		if stderr:
			if not "File exists" in stderr:
				common.internal_print("Error: adding client route: {0}".format(stderr), -1)
				sys.exit(-1)

		return 

def mac_set_intermediate_route(self, serverip, proxyip):
		common.internal_print("Changing route table for intermediate hop")

		ps = subprocess.Popen(["route", "delete", serverip, self.orig_default_gw], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		(stdout, stderr) = ps.communicate()
		if stderr:
			common.internal_print("Error: delete old route: {0}".format(stderr), -1)
			sys.exit(-1)

		ps = subprocess.Popen(["route", "add", "-net", proxyip, self.orig_default_gw, "255.255.255.255"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		(stdout, stderr) = ps.communicate()
		if stderr:
			if not "File exists" in stderr:
				common.internal_print("Error: adding server route: {0}".format(stderr), -1)
				sys.exit(-1)
		return 

def WIN_get_interface_index(self):
		iface_name = self.WIN_get_subinterface_name()
		ps = subprocess.Popen(["netsh", "interface", "ipv4", "show", "interfaces"], stdout=subprocess.PIPE,
			stderr=subprocess.PIPE)
		(stdout, stderr) = ps.communicate()


		if stderr != "":
			common.internal_print("Show interfaces. netsh failed: {0}".format(stdout), -1)
			sys.exit(-1)

		for line in stdout.split("\n"):
			if iface_name in line:
				i = 0
				while line[i:i+1] == " ":
					i += 1
				return int(line[i:].split(" ")[0])

		return -1 

def win_check_default_route(self):
		# get default gateway
		ps = subprocess.Popen(["route", "-4", "PRINT", "0.0.0.0"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		(stdout, stderr) = ps.communicate()
		if stderr:
			common.internal_print("Checking default route failed: {0}".format(stderr), -1)
			sys.exit(-1)

		# count default routes
		default_routes = 0
		for line in stdout[0:stdout.find("Persistent Routes:")].split("\n"):
			if "0.0.0.0" in line:
				default_routes += 1

		if not default_routes:
			common.internal_print("No default route. Please set up your routing before executing the tool", -1)
			sys.exit(-1)

		return 

def win_set_intermediate_route(self, serverip, proxyip):
		common.internal_print("Changing route table for intermediate hop")
		# delete original default route
		ps = subprocess.Popen(["route", "DELETE", serverip, self.orig_default_gw], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		(stdout, stderr) = ps.communicate()
		if stderr:
			common.internal_print("Delete server route failed: {0}".format(stderr), -1)
			sys.exit(-1)

		# add intermediate route
		ps = subprocess.Popen(["route", "ADD", proxyip, "MASK", "255.255.255.255", self.orig_default_gw], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		(stdout, stderr) = ps.communicate()
		if stderr:
			common.internal_print("Add intermediate route failed: {0}".format(stderr), -1)
			sys.exit(-1)

		return 

def win_restore_routes(self, serverip, clientip, ip):
		common.internal_print("Restoring default route")

		ps = subprocess.Popen(["route", "DELETE", serverip, self.orig_default_gw], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		(stdout, stderr) = ps.communicate()
		if stderr:
			common.internal_print("Delete server route failed: {0}".format(stderr), -1)
			sys.exit(-1)

		ps = subprocess.Popen(["route", "-p", "DELETE", "0.0.0.0", ip], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		(stdout, stderr) = ps.communicate()
		if stderr:
			common.internal_print("Delete default route failed: {0}".format(stderr), -1)
			sys.exit(-1)

		ps = subprocess.Popen(["route", "ADD", "0.0.0.0", "MASK", "0.0.0.0", self.orig_default_gw], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		(stdout, stderr) = ps.communicate()
		if stderr:
			common.internal_print("Add original default route failed: {0}".format(stderr), -1)
			sys.exit(-1)

		return 

def checkWebkitToPDF():
    try:
        subprocess.check_call("webkitToPDF", stderr=subprocess.PIPE, shell=True)
        return True
    except subprocess.CalledProcessError:
        return False 

def checkRSVGConvert():
    try:
        subprocess.check_call("rsvg-convert -v", stdout=subprocess.PIPE, shell=True)
        return True
    except subprocess.CalledProcessError:
        return False 

def checkInkscape():
    try:
        subprocess.check_call("inkscape --version", stdout=subprocess.PIPE, shell=True)
        return True
    except subprocess.CalledProcessError:
        return False 

def popen(cls, cmd, cwd=None, raises=False):
        '''
            Execute the given command string in a new process. Send data to stdin and
        read data from stdout and stderr, until end-of-file is reached.

        :param cls     : The class as implicit first argument.
        :param cwd     : If it is set, then the child's current directory will be change
                         to `cwd` before it is executed.
        :param raises  : If ``True`` and stderr has data, it raises an ``OSError`` exception.
        :returns       : The output of the given command; pair of (stdout, stderr).
        :rtype         : ``tuple``
        :raises OSError: If `raises` and stderr has data.
        '''
        parser   = lambda x: [] if x == '' else [y.strip() for y in x.strip().split('\n')]
        process  = subprocess.Popen(cmd, shell=True, universal_newlines=True,
                    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out, err = process.communicate()

        # .............................trim lines and remove the empty ones
        _stdout  = [x for x in parser(out) if bool(x)]
        _stderr  = [x for x in parser(err) if bool(x)]

        if _stderr and raises:
            raise OSError('\n'.join(_stderr))

        return _stdout, _stderr 

def _run_command(command):
    p = subprocess.Popen(command,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT)
    return iter(p.stdout.readline, b'') 

def source(script: str, cwd=None) -> dict:
    popen_args = {}
    if cwd is not None:
        popen_args['cwd'] = cwd

    import subprocess
    cmd = 'bash -c \'source %s; bash %s\'' % (script, print_env_sh_path)
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True, **popen_args)
    output = proc.communicate()[0]
    return dict(line.split('=', 1) for line in output.decode().split('\x00') if line)


# Creates the directory if no other file or directory with the same path exists