Python subprocess.Popen() Examples

def run_command(args, wait=False):

    try:
        if (wait):
            p = subprocess.Popen(
                args, 
                stdout = subprocess.PIPE)
            p.wait()
        else:
            p = subprocess.Popen(
                args, 
                stdin = None, stdout = None, stderr = None, close_fds = True)

        (result, error) = p.communicate()
        
    except subprocess.CalledProcessError as e:
        sys.stderr.write(
            "common::run_command() : [ERROR]: output = %s, error code = %s\n" 
            % (e.output, e.returncode))

    return result 

def cmd(command):
    result = Result()

    p = Popen(shlex.split(command), stdin=PIPE, stdout=PIPE, stderr=PIPE)
    (stdout, stderr) = p.communicate()

    result.exit_code = p.returncode
    result.stdout = stdout
    result.stderr = stderr
    result.command = command

    if p.returncode != 0:
        print 'Error executing command [%s]' % command
        print 'stderr: [%s]' % stderr
        print 'stdout: [%s]' % stdout

    return result 

def run_command(command, wait=False):

    try:
        if (wait):

            p = subprocess.Popen(
                [command], 
                stdout = subprocess.PIPE,
                shell = True)
            p.wait()
        else:
            p = subprocess.Popen(
                [command], 
                shell = True, 
                stdin = None, stdout = None, stderr = None, close_fds = True)

        (result, error) = p.communicate()
        
    except subprocess.CalledProcessError as e:
        sys.stderr.write(
            "common::run_command() : [ERROR]: output = %s, error code = %s\n" 
            % (e.output, e.returncode))

    return result 

def call(*popenargs, **kwargs):
        """Run command with arguments. Wait for command to complete. Sends
        output to logging module. The arguments are the same as for the Popen
        constructor."""

        from subprocess import Popen, PIPE

        kwargs['stdout'] = PIPE
        kwargs['stderr'] = PIPE

        p = Popen(*popenargs, **kwargs)
        stdout, stderr = p.communicate()

        if stdout:
            for line in stdout.strip().split("\n"):
                logger.info(line)

        if stderr:
            for line in stderr.strip().split("\n"):
                logger.error(line)

        return p.returncode 

def mathjax(s):
    with open("temp.log", "w") as f:
        f.write(s)

    p = Popen([app.config['mjpage'],
              '--dollars',
               '--output', "CommonHTML",
               '--fontURL',
               ("https://cdnjs.cloudflare.com/ajax/libs/"
                "mathjax/2.7.0/fonts/HTML-CSS")], stdout=PIPE, stdin=PIPE,
              stderr=PIPE)

    #filename = hashlib.sha256(s.encode('utf-8')).hexdigest()
    #with open(filename, 'w') as f:
    #    print(s, file=f)

    res = p.communicate(input=s.encode('utf-8'))
    out = res[0].decode('utf-8')
    err = res[1].decode('utf-8')

    soup = BeautifulSoup(out, 'html.parser')
    style = str(soup.style)
    body = "".join(str(s) for s in soup.body.children)

    return style, body 

def start_command(command):
    try:
        subp = subprocess.Popen(command, shell=True,
             stdin=None, stdout=None, stderr=None, close_fds=True)
        return '[+] Command successfully started.\n'
    except Exception as e:
        return '[!] {}\n'.format(str(e)) 

def store_revision_info(src_path, output_dir, arg_string):
  
    # Get git hash
    gitproc = Popen(['git', 'rev-parse', 'HEAD'], stdout = PIPE, cwd=src_path)
    (stdout, _) = gitproc.communicate()
    git_hash = stdout.strip()
  
    # Get local changes
    gitproc = Popen(['git', 'diff', 'HEAD'], stdout = PIPE, cwd=src_path)
    (stdout, _) = gitproc.communicate()
    git_diff = stdout.strip()
    
    # Store a text file in the log directory
    rev_info_filename = os.path.join(output_dir, 'revision_info.txt')
    with open(rev_info_filename, "w") as text_file:
        text_file.write('arguments: %s\n--------------------\n' % arg_string)
        text_file.write('git hash: %s\n--------------------\n' % git_hash)
        text_file.write('%s' % git_diff) 

def test_BadUserOrBadGroup(self):
        """Test that we read the correct domainname from the DMD file, the test domain
        should have been created by the test runner"""
        domainName = scatest.getTestDomainName()
        # Test that we don't already have a bound domain
        try:
            domMgr = self._root.resolve(scatest.getDomainMgrURI())
            self.assertEqual(domMgr, None)
        except CosNaming.NamingContext.NotFound:
            pass # This exception is expected

        args = ["../../control/framework/nodeBooter","-D","-debug", "9","--nopersist",'--user=domuser']
        nb = Popen( args, cwd=scatest.getSdrPath(), stderr=PIPE, stdout=PIPE)
        self.assertNotEqual(nb.stderr.read().find('Separator must be a space'),-1)

        args = ["../../control/framework/nodeBooter","-D","-debug", "9","--nopersist",'--group=somegroup']
        nb = Popen( args, cwd=scatest.getSdrPath(), stderr=PIPE, stdout=PIPE)
        self.assertNotEqual(nb.stderr.read().find('Separator must be a space'),-1) 

def plot(self):
        if _domainless._DEBUG == True:
            print "Plot:plot()"

        # Error checking before launching plot
        if self.usesPortIORString == None:
            raise AssertionError, "Plot:plot() ERROR - usesPortIORString not set ... must call connect() on this object from another component"
        if self._usesPortName == None:
            raise AssertionError, "Plot:plot() ERROR - usesPortName not set ... must call connect() on this object from another component"
        if self._dataType == None:
            raise AssertionError, "Plot:plot() ERROR - dataType not set ... must call connect() on this object from another component"

        plotCommand = str(self._eclipsePath) + "/bin/plotter.sh -portname " + str(self._usesPortName) + " -repid " + str(self._dataType) + " -ior " + str(self.usesPortIORString)
        if _domainless._DEBUG == True:
            print "Plot:plotCommand " + str(plotCommand)
        args = _shlex.split(plotCommand)
        if _domainless._DEBUG == True:
            print "Plot:args " + str(args)
        try:
            dev_null = open('/dev/null','w')
            sub_process = _subprocess.Popen(args,stdout=dev_null,preexec_fn=_os.setpgrp)
            pid = sub_process.pid
            self._processes[pid] = sub_process
        except Exception, e:
            raise AssertionError, "Plot:plot() Failed to launch plotting due to %s" % ( e) 

def win64_available(self):

        wine_bin, \
        wineserver_bin, \
        wine_lib = self.get_wine_bin_path()

        dev_null = open(os.devnull, 'w')
        try:
            proc = subprocess.Popen([wine_bin + '64'], stdout=dev_null, \
                stdin=subprocess.PIPE, stderr=subprocess.STDOUT)
            dev_null.close()
            stdoutdata, stderrdata = proc.communicate()
            if proc.returncode == 1:
                self.combobox_winearch.set_visible(True)
                return True
            else:
                self.combobox_winearch.set_visible(False)
                self.winearch = 'win32'
                return False
        except:
            self.combobox_winearch.set_visible(False)
            self.winearch = 'win32'
            return False 

def index():
    if request.args.get('code'):
        unlock_code = request.args.get('code')
        # unlock, new password
        re = s.query(RecoveryEmail).filter(RecoveryEmail.password_code==unlock_code).one_or_none()
        if re:
            jid = re.jid
            re.password_code = None
            s.merge(re)
            s.commit()
            # set new password and send email
            email_address = re.email
            password = ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.ascii_uppercase + string.digits) for _ in range(10))
            p = subprocess.Popen(['/usr/bin/prosodyctl', 'passwd', jid], stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.STDOUT)
            args = bytes("%s\n%s\n" % (password, password), encoding='utf8')
            p.communicate(args)
            sendMail(email_address, 'new password', password)
            content = render_template('success.html', message='password was sent')
        else:
            content = render_template('error.html', message='link invalid')
    else:
        content = render_template('index.html')
    return content 

def _compile_proto(full_path, dest):
    'Helper to compile protobuf files'
    proto_path = os.path.dirname(full_path)
    protoc_args = [find_protoc(),
                   '--python_out={}'.format(dest),
                   '--proto_path={}'.format(proto_path),
                   full_path]
    proc = subprocess.Popen(protoc_args, stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE)
    try:
        outs, errs = proc.communicate(timeout=5)
    except subprocess.TimeoutExpired:
        proc.kill()
        outs, errs = proc.communicate()
        return False

    if proc.returncode != 0:
        msg = 'Failed compiling "{}": \n\nstderr: {}\nstdout: {}'.format(
            full_path, errs.decode('utf-8'), outs.decode('utf-8'))
        raise BadProtobuf(msg)

    return True 

def spawn(self, lines, additional_args = [ '-p', ''], width = None):
        (mouse_x, mouse_y) = get_mouse_location()
        if not width:
            width = 100 # some default width
        width = max(width, 101) # width has to be 100 at least (rofi restriction)
        # first, compute the top left corner of the menu
        menu_x = min(max(mouse_x - width/2, self.x), self.x + self.panel_width - width)
        menu_y = self.y
        # then, specify these coordinates relative to the mouse cursor
        menu_x -= mouse_x
        menu_y -= mouse_y
        # compile rofi arguments
        cmd = ['rofi', '-dmenu', '-sep' , '\\0' ]
        cmd += ['-monitor', '-3' ] # position relative to mouse cursor
        cmd += ['-layout', '1' ] # specify top left corner of the menu
        cmd += ['-width', str(width) ]
        cmd += ['-xoffset', str(menu_x), '-yoffset', str(menu_y) ]
        cmd += self.rofi_args
        cmd += additional_args
        rofi = subprocess.Popen(cmd,stdout=subprocess.PIPE,stdin=subprocess.PIPE)
        for i in lines:
            rofi.stdin.write(i.encode('utf-8'))
            rofi.stdin.write(struct.pack('B', 0))
        rofi.stdin.close()
        rofi.wait() 

def communicate(input, coro=None):
    if platform.system() == 'Windows':
        # asyncfile.Popen must be used instead of subprocess.Popen
        pipe = asyncoro.asyncfile.Popen([r'\cygwin64\bin\sha1sum.exe'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    else:
        pipe = subprocess.Popen(['sha1sum'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    # convert pipe to asynchronous version
    async_pipe = asyncoro.asyncfile.AsyncPipe(pipe)
    # 'communicate' takes either the data or file descriptor with data
    # (if file is too large to read in full) as input
    input = open(input)
    stdout, stderr = yield async_pipe.communicate(input)
    print('communicate sha1sum: %s' % stdout) 

def custom_feeder(input, coro=None):
    def write_proc(fin, pipe, coro=None):
        while True:
            data = yield os.read(fin.fileno(), 8*1024)
            if not data:
                break
            n = yield pipe.write(data, full=True)
            assert n == len(data)
        fin.close()
        pipe.stdin.close()

    def read_proc(pipe, coro=None):
        # output from sha1sum is small, so read until EOF
        data = yield pipe.stdout.read()
        pipe.stdout.close()
        raise StopIteration(data)

    if platform.system() == 'Windows':
        # asyncfile.Popen must be used instead of subprocess.Popen
        pipe = asyncoro.asyncfile.Popen([r'\cygwin64\bin\sha1sum.exe'],
                                        stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    else:
        pipe = subprocess.Popen(['sha1sum'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)

    async_pipe = asyncoro.asyncfile.AsyncPipe(pipe)
    reader = asyncoro.Coro(read_proc, async_pipe)
    writer = asyncoro.Coro(write_proc, open(input), async_pipe)
    stdout = yield reader.finish()
    print('     feeder sha1sum: %s' % stdout)

# asyncoro.logger.setLevel(asyncoro.Logger.DEBUG)

# simpler version using 'communicate' 

def communicate(input, coro=None):
    if platform.system() == 'Windows':
        # asyncfile.Popen must be used instead of subprocess.Popen
        pipe = asyncoro.asyncfile.Popen([r'\cygwin64\bin\sha1sum.exe'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    else:
        pipe = subprocess.Popen(['sha1sum'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    # convert pipe to asynchronous version
    async_pipe = asyncoro.asyncfile.AsyncPipe(pipe)
    # 'communicate' takes either the data or file descriptor with data
    # (if file is too large to read in full) as input
    input = open(input)
    stdout, stderr = yield async_pipe.communicate(input)
    print('communicate sha1sum: %s' % stdout) 

def custom_feeder(input, coro=None):
    def write_proc(fin, pipe, coro=None):
        while True:
            data = yield os.read(fin.fileno(), 8*1024)
            if not data:
                break
            n = yield pipe.write(data, full=True)
            assert n == len(data)
        fin.close()
        pipe.stdin.close()

    def read_proc(pipe, coro=None):
        # output from sha1sum is small, so read until EOF
        data = yield pipe.stdout.read()
        pipe.stdout.close()
        raise StopIteration(data)

    if platform.system() == 'Windows':
        # asyncfile.Popen must be used instead of subprocess.Popen
        pipe = asyncoro.asyncfile.Popen([r'\cygwin64\bin\sha1sum.exe'],
                                        stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    else:
        pipe = subprocess.Popen(['sha1sum'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)

    async_pipe = asyncoro.asyncfile.AsyncPipe(pipe)
    reader = asyncoro.Coro(read_proc, async_pipe)
    writer = asyncoro.Coro(write_proc, open(input), async_pipe)
    stdout = yield reader.finish()
    print('     feeder sha1sum: %s' % stdout)

# asyncoro.logger.setLevel(asyncoro.Logger.DEBUG)

# simpler version using 'communicate' 

def terminate(self):
            """Close pipe and terminate child process.
            """
            self.close()
            super(Popen, self).terminate() 

def poll(self):
        """Similar to 'poll' of Popen.
        """
        if self.last:
            return self.last.poll()
        elif self.first:
            return self.first.poll() 

def custom_feeder(input, coro=None):
    def write_proc(fin, pipe, coro=None):
        while True:
            data = yield os.read(fin.fileno(), 8*1024)
            if not data:
                break
            n = yield pipe.write(data, full=True)
            assert n == len(data)
        fin.close()
        pipe.stdin.close()

    def read_proc(pipe, coro=None):
        # output from sha1sum is small, so read until EOF
        data = yield pipe.stdout.read()
        pipe.stdout.close()
        raise StopIteration(data)

    if platform.system() == 'Windows':
        # asyncfile.Popen must be used instead of subprocess.Popen
        pipe = asyncoro.asyncfile.Popen([r'\cygwin64\bin\sha1sum.exe'],
                                        stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    else:
        pipe = subprocess.Popen(['sha1sum'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)

    async_pipe = asyncoro.asyncfile.AsyncPipe(pipe)
    reader = asyncoro.Coro(read_proc, async_pipe)
    writer = asyncoro.Coro(write_proc, open(input), async_pipe)
    stdout = yield reader.finish()
    print('     feeder sha1sum: %s' % stdout)

# asyncoro.logger.setLevel(asyncoro.Logger.DEBUG)

# simpler version using 'communicate' 

def terminate(self):
            """Close pipe and terminate child process.
            """
            self.close()
            super(Popen, self).terminate() 

def __init__(self, first, last=None):
        """'first' is a Popen object. 'last', if given, is another
        Popen object that is the end of the joints to 'first'.

        'write' operations send data to first's stdin and 'read'
        operations get data from last's stdout/stderr.
        """
        if not last:
            last = first
        self.first = first
        self.last = last
        if platform.system() == 'Windows':
            if not isinstance(first, Popen) or not isinstance(last, Popen):
                raise ValueError('argument must be asyncfile.Popen object')
            if first.stdin:
                self.stdin = first.stdin
            else:
                self.stdin = None
            if last.stdout:
                self.stdout = last.stdout
            else:
                self.stdout = None
            if last.stderr:
                self.stderr = last.stderr
            else:
                self.stderr = None
        else:
            if not isinstance(first, subprocess.Popen) or not isinstance(last, subprocess.Popen):
                raise ValueError('argument must be subprocess.Popen object')
            if first.stdin:
                self.stdin = AsyncFile(first.stdin)
            else:
                self.stdin = None
            if last.stdout:
                self.stdout = AsyncFile(last.stdout)
            else:
                self.stdout = None
            if last.stderr:
                self.stderr = AsyncFile(last.stderr)
            else:
                self.stderr = None 

def poll(self):
        """Similar to 'poll' of Popen.
        """
        if self.last:
            return self.last.poll()
        elif self.first:
            return self.first.poll() 

def execute(cmd, success_msg="", failure_msg="", exitcode=-1):
    """
    Generic wrapper to execute the CLI commands. Returns Output if success.
    On success it can print message in stdout if specified.
    On failure, exits after writing to stderr.
    """
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = p.communicate()
    if p.returncode == 0:
        if success_msg:
            output_ok(success_msg)
        return out
    else:
        err_msg = err if err else out
        output_notok(failure_msg, err=err_msg, exitcode=exitcode) 

def get_glusterd_workdir():
    """
    Command to get Glusterd working dir. If failed returns the
    default directory /var/lib/glusterd
    """
    p = subprocess.Popen(["gluster", "system::", "getwd"],
                         stdout=subprocess.PIPE)

    out, _ = p.communicate()

    if p.returncode == 0:
        return out.strip()
    else:
        return DEFAULT_GLUSTERD_WORKDIR 

def parse_tox(self):
        proc = subprocess.Popen(
            "tox -l", shell=True, stdout=subprocess.PIPE, cwd=self.cwd)
        self.tox_lines = proc.stdout.read().strip().split('\n')
        self.parse_python_versions() 

def run_command(command):
    try:
        subp = subprocess.Popen(command,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
        subp_output, errors = subp.communicate()
        if not errors:
            if subp_output == '':
                return '[+] Command successfully executed.\n'
            else:
                return subp_output
        return "[!] {}".format(errors)
    except KeyboardInterrupt:
        print "Terminated command." 

def check_dep():
    dep = subprocess.Popen('wmic OS get DataExecutionPrevention_SupportPolicy',shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
    dep_mode, errors = dep.communicate()
    if not errors and dep_mode:
        if "0" in dep_mode:
            return "DEP is off for the whole system."
        elif "1" in dep_mode:
            return "Full DEP coverage for the whole system with no exceptions."
        elif "2" in dep_mode:
            return "DEP is limited to Windows system binaries."
        elif "3" in dep_mode:
            return "DEP is on for all programs and services."
    else:
        return errors 

def checklocalfw():
    print("Getting Windows Built in Firewall configuration...")
    fw = subprocess.Popen('netsh advfirewall show all state',shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
    fw_mode, errors = fw.communicate()
    if not errors and fw_mode:
        return fw_mode
    else:
        return errros 

def run_command(command):
    try:
        subp = subprocess.Popen(command,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
        subp_output, errors = subp.communicate()
        if not errors:
            if subp_output == '':
                return '[+] Command successfully executed.\n'
            else:
                return subp_output
        return "[!] {}".format(errors)
    except KeyboardInterrupt:
        print "Terminated command." 

def run_command(command):
    try:
        subp = subprocess.Popen(command,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
        subp_output, errors = subp.communicate()
        if not errors:
            if subp_output == '':
                return '[+] Command successfully executed.\n'
            else:
                return subp_output
        return "[!] {}\n".format(errors)
    except KeyboardInterrupt:
        print "Terminated command." 

def get_command_result(cmd):
    p = subprocess.Popen(cmd.split(Res.white_space),
                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = p.communicate()
    return err, out 

def _stringify(obj):
    """Turn ``obj`` into a string for `Popen`."""
    if isinstance(obj, str):
        return obj

    if isinstance(obj, unicode):
        return obj.encode('utf-8')

    return str(obj) 

def _call_security(self, action, service, account, *args):
        """Call ``security`` CLI program that provides access to keychains.

        May raise `PasswordNotFound`, `PasswordExists` or `KeychainError`
        exceptions (the first two are subclasses of `KeychainError`).

        :param action: The ``security`` action to call, e.g.
                           ``add-generic-password``
        :type action: ``unicode``
        :param service: Name of the service.
        :type service: ``unicode``
        :param account: name of the account the password is for, e.g.
            "Pinboard"
        :type account: ``unicode``
        :param password: the password to secure
        :type password: ``unicode``
        :param *args: list of command line arguments to be passed to
                      ``security``
        :type *args: `list` or `tuple`
        :returns: ``(retcode, output)``. ``retcode`` is an `int`, ``output`` a
                  ``unicode`` string.
        :rtype: `tuple` (`int`, ``unicode``)

        """
        cmd = ['security', action, '-s', service, '-a', account] + list(args)
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)
        stdout, _ = p.communicate()
        if p.returncode == 44:  # password does not exist
            raise PasswordNotFound()
        elif p.returncode == 45:  # password already exists
            raise PasswordExists()
        elif p.returncode > 0:
            err = KeychainError('Unknown Keychain error : %s' % stdout)
            err.retcode = p.returncode
            raise err
        return stdout.strip().decode('utf-8') 

def run_script(self, script):
        fd, self.tmp_script_filename = tempfile.mkstemp()
        os.close(fd)
        f = open(self.tmp_script_filename, 'w')
        f.write(script)
        f.close()
        os.chmod(self.tmp_script_filename, 0o744)
        p = subprocess.Popen(
            self.tmp_script_filename, shell=True, stdin=subprocess.PIPE,
            stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
        self.process = p
        self.stream = p.stdout 

def run(daemon):
        if daemon:
            pid_file = './sensor21.pid'
            if os.path.isfile(pid_file):
                pid = int(open(pid_file).read())
                os.remove(pid_file)
                try:
                    p = psutil.Process(pid)
                    p.terminate()
                except:
                    pass
            try:
                p = subprocess.Popen(['python3', 'sensor21-server.py'])
                open(pid_file, 'w').write(str(p.pid))
            except subprocess.CalledProcessError:
                raise ValueError("error starting sensor21-server.py daemon")
        else:
            print("Server running...")
            app.run(host='::', port=5002) 

def restart_pigpiod(self):
        """
        Restarts the pigpiod daemon if there is a connection
        timeout.
        Used only with software I2C.
        """
        if self.i2c_type == 1:
            # Exit resources and kill the daemon
            self.i2c_channel.close_bus()
            self.i2c_channel.stop()
            pigpio_kill_string = "sudo pkill pigpiod"
            subprocess.Popen(pigpio_kill_string, shell=True)
            time.sleep(5)

            # Restart the daemon & reinitalize connections
            pigpio_start_string = "sudo pigpiod"
            start = subprocess.Popen(pigpio_start_string, shell=True)
            time.sleep(5)

            self.i2c_channel = PigpioI2CBitBang()
            self.i2c_channel.open_bus()
            self.mpl3115a2 = MPL3115A2(self.i2c_channel) 

def ProcParallel(cmd,i,njobs,email=False,inName='logIn.log',outName='logOut.log',errName='logErr.log'):
    global firstRun
    global procs
    if firstRun:
        logIn = open(inName, 'w')
        logOut = open(outName, 'w')
        logErr = open(errName, 'w')
        firstRun = False
    else:
        logIn = open(inName, 'a')
        logOut = open(outName, 'a')
        logErr = open(errName, 'a')

    inl = os.tmpfile()
    outl = os.tmpfile()
    errl = os.tmpfile()
    print '\033[1;33m-> %s\033[0m [%i of %i]' %(cmd,i+1,njobs)
    start = datetime.datetime.now()
    if email==False:
        proc = Popen(cmd,shell=True,stdin=inl,stdout=outl,stderr=errl)
    else:
        proc = Popen(_Notifier(cmd,i+1,njobs,email),shell=True,stdin=inl,stdout=outl,stderr=errl)
    procs.append((start,cmd,proc,inl,outl,errl))

    if i==njobs-1:
        for j, (start, name, proc, inl, outl, errl) in enumerate(procs):
            proc.wait()
            end = datetime.datetime.now()
            print 'Process %i of %i completed. Time: %s' %(j+1,njobs,end-start)
            _Temp2Perm(inl,logIn,name,start,end)
            _Temp2Perm(outl,logOut,name,start,end)
            if errl.tell() != 0:
                print '\033[1;31mErrors have occurred!\033[0m (check %s)' %(logErr.name)
                _Temp2Perm(errl,logErr,cmd,start,end)
        firstRun = True
        procs = [] 

def run(self, timeout):
        def target():
            self.process = subprocess.Popen(self.cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            self.output, self.error = self.process.communicate()

        thread = threading.Thread(target=target)
        thread.start()
        thread.join(timeout)

        if thread.is_alive():
            self.process.kill()
            return (False, "timeout", self.process.pid)
        return (self.process.returncode, self.output, self.error) 

def run(self, timeout):
        def target():
            self.process = subprocess.Popen(self.cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            self.output, self.error = self.process.communicate()

        thread = threading.Thread(target=target)
        thread.start()
        thread.join(timeout)

        if thread.is_alive():
            self.process.kill()
            return (False, "timeout", self.process.pid)
        return (self.process.returncode, self.output, self.error) 

def uname(self):
        p = subprocess.Popen(shlex.split('uname -i'), stdout=subprocess.PIPE, close_fds=True)
        hwplatform = p.stdout.read().strip()
        p.wait()

        uname = platform.uname()
        return {
            'kernel_name': uname[0],
            'node': uname[1],
            'kernel_release': uname[2],
            'kernel_version': uname[3],
            'machine': uname[4],
            'processor': uname[5],
            'platform': hwplatform,
        } 

def partinfo(self, uuid=None, devname=None):
        """Read partition info including uuid and filesystem.
        
        You can specify uuid or devname to get the identified partition info.
        If no argument provided, all partitions will return.
        
        We read info from /etc/blkid/blkid.tab instead of call blkid command.
        REF: http://linuxconfig.org/how-to-retrieve-and-change-partitions-universally-unique-identifier-uuid-on-linux
        """
        blks = {}
        p = subprocess.Popen(shlex.split('/sbin/blkid'), stdout=subprocess.PIPE, close_fds=True)
        p.stdout.read()
        p.wait()

        # OpenVZ may not have this file
        if not os.path.exists('/etc/blkid/blkid.tab'): return None

        with open('/etc/blkid/blkid.tab') as f:
            for line in f:
                dom = parseString(line).documentElement
                _fstype = dom.getAttribute('TYPE')
                _uuid = dom.getAttribute('UUID')
                _devname = dom.firstChild.nodeValue.replace('/dev/', '')
                partinfo = {
                    'name': _devname,
                    'fstype': _fstype,
                    'uuid': _uuid,
                }
                if uuid and uuid == _uuid:
                    return partinfo
                elif devname and devname == _devname:
                    return partinfo
                else:
                    blks[_devname] = partinfo
        if uuid or devname:
            return None
        else:
            return blks 

def play(self, streamUrl):
        self.close()
        opts = self._buildStartOpts(streamUrl)
        self.process = subprocess.Popen(opts, shell=False,
                            stdin=subprocess.PIPE,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE)
        if logger.isEnabledFor(logging.DEBUG):
            logger.debug("Player started") 

def get_sign_command(self, filename, signer, sign_password,
                         keystore=None):
        """
        Return a suitable command for signing a file.

        :param filename: The pathname to the file to be signed.
        :param signer: The identifier of the signer of the file.
        :param sign_password: The passphrase for the signer's
                              private key used for signing.
        :param keystore: The path to a directory which contains the keys
                         used in verification. If not specified, the
                         instance's ``gpg_home`` attribute is used instead.
        :return: The signing command as a list suitable to be
                 passed to :class:`subprocess.Popen`.
        """
        cmd = [self.gpg, '--status-fd', '2', '--no-tty']
        if keystore is None:
            keystore = self.gpg_home
        if keystore:
            cmd.extend(['--homedir', keystore])
        if sign_password is not None:
            cmd.extend(['--batch', '--passphrase-fd', '0'])
        td = tempfile.mkdtemp()
        sf = os.path.join(td, os.path.basename(filename) + '.asc')
        cmd.extend(['--detach-sign', '--armor', '--local-user',
                    signer, '--output', sf, filename])
        logger.debug('invoking: %s', ' '.join(cmd))
        return cmd, sf 

def run_command(self, cmd, input_data=None):
        """
        Run a command in a child process , passing it any input data specified.

        :param cmd: The command to run.
        :param input_data: If specified, this must be a byte string containing
                           data to be sent to the child process.
        :return: A tuple consisting of the subprocess' exit code, a list of
                 lines read from the subprocess' ``stdout``, and a list of
                 lines read from the subprocess' ``stderr``.
        """
        kwargs = {
            'stdout': subprocess.PIPE,
            'stderr': subprocess.PIPE,
        }
        if input_data is not None:
            kwargs['stdin'] = subprocess.PIPE
        stdout = []
        stderr = []
        p = subprocess.Popen(cmd, **kwargs)
        # We don't use communicate() here because we may need to
        # get clever with interacting with the command
        t1 = Thread(target=self._reader, args=('stdout', p.stdout, stdout))
        t1.start()
        t2 = Thread(target=self._reader, args=('stderr', p.stderr, stderr))
        t2.start()
        if input_data is not None:
            p.stdin.write(input_data)
            p.stdin.close()

        p.wait()
        t1.join()
        t2.join()
        return p.returncode, stdout, stderr 

def get_verify_command(self, signature_filename, data_filename,
                           keystore=None):
        """
        Return a suitable command for verifying a file.

        :param signature_filename: The pathname to the file containing the
                                   signature.
        :param data_filename: The pathname to the file containing the
                              signed data.
        :param keystore: The path to a directory which contains the keys
                         used in verification. If not specified, the
                         instance's ``gpg_home`` attribute is used instead.
        :return: The verifying command as a list suitable to be
                 passed to :class:`subprocess.Popen`.
        """
        cmd = [self.gpg, '--status-fd', '2', '--no-tty']
        if keystore is None:
            keystore = self.gpg_home
        if keystore:
            cmd.extend(['--homedir', keystore])
        cmd.extend(['--verify', signature_filename, data_filename])
        logger.debug('invoking: %s', ' '.join(cmd))
        return cmd 

def run_command(self, cmd, **kwargs):
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE, **kwargs)
        t1 = threading.Thread(target=self.reader, args=(p.stdout, 'stdout'))
        t1.start()
        t2 = threading.Thread(target=self.reader, args=(p.stderr, 'stderr'))
        t2.start()
        p.wait()
        t1.join()
        t2.join()
        if self.progress is not None:
            self.progress('done.', 'main')
        elif self.verbose:
            sys.stderr.write('done.\n')
        return p 

def _get_lsb_release_info(self):
        """
        Get the information items from the lsb_release command output.

        Returns:
            A dictionary containing all information items.
        """
        cmd = 'lsb_release -a'
        process = subprocess.Popen(
            cmd,
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
        stdout, stderr = process.communicate()
        stdout, stderr = stdout.decode('utf-8'), stderr.decode('utf-8')
        code = process.returncode
        if code == 0:
            content = stdout.splitlines()
            return self._parse_lsb_release_content(content)
        elif code == 127:  # Command not found
            return {}
        else:
            if sys.version_info[:2] >= (3, 5):
                raise subprocess.CalledProcessError(code, cmd, stdout, stderr)
            elif sys.version_info[:2] >= (2, 7):
                raise subprocess.CalledProcessError(code, cmd, stdout)
            elif sys.version_info[:2] == (2, 6):
                raise subprocess.CalledProcessError(code, cmd) 

def exec_cmd(cmd, **kwds):
    """
    Execute arbitrary commands as sub-processes.
    """
    stdin = kwds.get('stdin', None)
    stdin_flag = None
    if not stdin is None:
        stdin_flag = subprocess.PIPE
    proc = subprocess.Popen(
        cmd,
        stdin=stdin_flag,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE)
    stdout, stderr = proc.communicate(stdin)
    return (proc.returncode, stdout, stderr)


#=======================================================================
# Extend the Milter Class (where the email is captured)
#======================================================================= 

def scrapeDoi(url):
    env = os.environ.copy()
    cmd_line = ['timeout', '30s', 'google-chrome-unstable', '--headless', '--dump-dom', url]
    p = subprocess.Popen(cmd_line, stdin=subprocess.PIPE,
                         stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                         env=env)
    out, err = p.communicate()
    if p.returncode:
        print('UTOH')
        return None
    elif b'ERROR:headless_shell.cc' in out:
        print(out)
        raise IOError('Something is wrong...')
    qurl = quote(url, '')
    if len(qurl) > 200:
        qurl = qurl[:200]
    with open(os.path.expanduser(f'~/files/scibot/{qurl}'), 'wb') as f:
        f.write(out)
    both = BeautifulSoup(out, 'lxml')
    doi = getDoi(both, both)
    return doi