Python subprocess.run() Examples

def sync_assets(c):
    subprocess.run(
        [
            "rsync",
            "-r",
            "-e",
            "ssh -p {port}".format(port=c.conn.port),
            "--exclude",
            "*.map",
            "--exclude",
            "*.swp",
            "static/dist",
            "{user}@{host}:{path}".format(
                host=c.conn.host,
                user=c.conn.user,
                path=os.path.join(c.conn.project_root, 'static'),
            ),
        ]
    )


# Environment handling stuff
############################ 

def clean_trial(src_loc: Path, test_cmds: List[str]) -> timedelta:
    """Remove all existing cache files and run the test suite.

    Args:
        src_loc: the directory of the package for cache removal, may be a file
        test_cmds: test running commands for subprocess.run()

    Returns:
        None

    Raises:
        BaselineTestException: if the clean trial does not pass from the test run.
    """
    cache.remove_existing_cache_files(src_loc)

    LOGGER.info("Running clean trial")

    # clean trial will show output all the time for diagnostic purposes
    start = datetime.now()
    clean_run = subprocess.run(test_cmds, capture_output=False)
    end = datetime.now()

    if clean_run.returncode != 0:
        raise BaselineTestException(
            f"Clean trial does not pass, mutant tests will be meaningless.\n"
            f"Output: {str(clean_run.stdout)}"
        )

    return end - start


####################################################################################################
# MUTATION SAMPLE GENERATION
#################################################################################################### 

def handle_dependencies(self, file_path, is_module):
        args = f"cargo run -p compiler -- -l {file_path}"
        if is_module:
            args += " -m"
        output = subprocess.check_output(args.split(), cwd="../libra/")
        access_paths = json.loads(output)
        dependencies = []
        for path in access_paths:
            if not Address.equal_address(path['address'], libra.AccountConfig.core_code_address()):
                amap = self.grpc_client.get_account_state(path['address']).ordered_map
                code = amap[bytes(path['path'])]
                if code:
                    dependencies.append(code)
        if not dependencies:
            return None
        tmp = NamedTemporaryFile('w+t')
        with open(tmp.name, 'wt') as f:
            json.dump(dependencies, f)
        return tmp 

def create_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="Pytype/typeshed tests.")
    parser.add_argument("-n", "--dry-run", action="store_true", default=False, help="Don't actually run tests")
    # Default to '' so that symlinking typeshed subdirs in cwd will work.
    parser.add_argument("--typeshed-location", type=str, default="", help="Path to typeshed installation.")
    # Set to true to print a stack trace every time an exception is thrown.
    parser.add_argument(
        "--print-stderr", action="store_true", default=False, help="Print stderr every time an error is encountered."
    )
    # We need to invoke python2.7 and 3.6.
    parser.add_argument("--python27-exe", type=str, default="python2.7", help="Path to a python 2.7 interpreter.")
    parser.add_argument("--python36-exe", type=str, default="python3.6", help="Path to a python 3.6 interpreter.")
    parser.add_argument(
        "files", metavar="FILE", type=str, nargs="*", help="Files or directories to check. (Default: Check all files.)",
    )
    return parser 

def req_common(self):
        # Now we have downloaded new code, install requirements
        logger.debug("Installing new requirements...")
        try:
            subprocess.run([sys.executable, "-m", "pip", "install", "-r",
                            os.path.join(os.path.dirname(utils.get_base_dir()), "requirements.txt"), "--user"])
        except subprocess.CalledProcessError:
            logger.exception("Req install failed") 

def exec_privilege_escalated_command(exec_string, *bindmounts):
    """Function to simulate sudo <command> by bind-mounting affected paths
       though docker.
       bindmounts is a list of `-v` style docker args
       e.g. `/home/user/elasticsearch-docker/tests/datadir1:/datadir1`
    """
    bind_mount_cli_args = []
    for bindarg in bindmounts:
        bind_mount_cli_args.append('-v')
        bind_mount_cli_args.append(bindarg)

    proc = run(
        ['docker', 'run', '--rm'] +
        bind_mount_cli_args +
        ['alpine', '/bin/sh', '-c', exec_string],
        stdout=PIPE)
    return proc 

def fetch_db(c, destination="."):
    """
    Dump the database on the remote host and retrieve it locally.

    The destination parameter controls where the dump should be stored locally.
    """
    dump_path = c.conn.dump_db("~")
    filename = os.path.basename(dump_path)

    subprocess.run(
        [
            "scp",
            "-P",
            str(c.conn.port),
            "{user}@{host}:{directory}".format(
                user=c.conn.user, host=c.conn.host, directory=dump_path
            ),
            destination,
        ]
    )
    c.conn.run("rm %s" % dump_path)

    return os.path.join(destination, filename) 

def open_in_gmsh(fn, new_thread=False):
    ''' Opens the mesh in gmsh

    Parameters
    ------------
    fn: str
        Name of mesh file
    new_thread: bool
        Wether to open gmsh in a new thread. Defaut: False

    '''
    gmsh_bin = path2bin('gmsh')
    if new_thread:
        t = threading.Thread(target=subprocess.run,
                             args=([gmsh_bin, fn], ),
                             kwargs={'check': True})
        t.daemon = False  # thread dies with the program
        t.start()
    else:
        subprocess.run([gmsh_bin, fn], check=True) 

def extract(iid, bindir):
    print('Extracting binaries......')
    query = '''select filename from object_to_image where iid=''' + iid + ''' and score>0 and (mime='application/x-executable; charset=binary' or mime='application/x-object; charset=binary' or mime='application/x-sharedlib; charset=binary') order by score DESC;'''
    wanted = dbquery(query)
    wanted = reduce((lambda a, b: a + b), wanted)
    wanted = map((lambda a: '.' + a), wanted)
    wanted = reduce((lambda a, b: a + ' ' + b), wanted)
    cmd = 'tar xf ' + bindir + '/../../../../images/' + iid + '.tar.gz -C ' + bindir + ' ' + wanted
    subprocess.run([cmd], shell=True)

    print('Extracting library links......')
    query = '''select filename from object_to_image where iid=''' + iid + ''' and regular_file='f';'''
    wanted = dbquery(query)
    wanted = reduce((lambda a, b: a + b), wanted)
    wanted = filter((lambda a: 'lib' in a), wanted)
    wanted = map((lambda a: '.' + a), wanted)
    wanted = reduce((lambda a, b: a + ' ' + b), wanted)
    cmd = 'tar xf ' + bindir + '/../../../../images/' + iid + '.tar.gz -C ' + bindir + ' ' + wanted
    subprocess.run([cmd], shell=True) 

def check_can_ping_all_leaves(self):
        step = "This leaf can ping all other leaves"
        success = True
        for pod in self.group.fabric.pods:
            for other_leaf_node in pod.nodes_by_layer[LEAF_LAYER]:
                if other_leaf_node == self:
                    continue
                for from_address in self.lo_addresses:
                    for to_address in other_leaf_node.lo_addresses:
                        result = subprocess.run(['ip', 'netns', 'exec', self.ns_name, 'ping', '-f',
                                                 '-W1', '-c10', '-I', from_address, to_address],
                                                stdout=subprocess.PIPE)
                        if result.returncode != 0:
                            error = 'Ping from {} {} to {} {} failed'.format(self.name,
                                                                             from_address,
                                                                             other_leaf_node.name,
                                                                             to_address)
                            self.report_check_result(step, False, error)
                            success = False
        if success:
            self.report_check_result(step)
        return success 

def exploit_shell(target, eid, outfile=None):
    print("Executing shell command...")

    # create log file for this shell command execution
    if outfile:
        outfile = outfile % {'exploit':eid}

    with smart_open(outfile, 'w') as f:
        ret = subprocess.run(SHELL_EXPLOITS[eid][0] % {'target': target},
                              stderr=f, stdout=f, shell=True).returncode
        # always run verification command if available; do not attempt early
        # termination if the first command appears to fail
        # this fixes e.g. 203, which crashes the HTTP server and causes curl to
        # return CURLE_GOT_NOTHING (52)
        if SHELL_EXPLOITS[eid][1]:
            ret = subprocess.run(SHELL_EXPLOITS[eid][1] % \
            {'target':target, 'output':outfile}, stderr=f, stdout=f, shell=True).returncode
        f.write("\nResult: %d" % ret) 

def ping(ns_name, source_lo_addr, dest_lo_addr):
    try:
        result = subprocess.run(['ip', 'netns', 'exec', ns_name, 'ping', '-f', '-W1',
                                 '-c{}'.format(PING_PACKTES),
                                 '-I', source_lo_addr, dest_lo_addr],
                                stdout=subprocess.PIPE)
    except FileNotFoundError:
        fatal_error('"ping" command not found')
    output = result.stdout.decode('ascii')
    lines = output.splitlines()
    for line in lines:
        if "packets transmitted" in line:
            split_line = line.split()
            packets_transmitted = int(split_line[0])
            packets_received = int(split_line[3])
            return (packets_transmitted, packets_received)
    fatal_error('Could not determine ping statistics for namespace "{}"'.format(ns_name))
    return None  # Never reached 

def check_process_running(self):
        step = "RIFT process is running"
        result = subprocess.run(['ip', 'netns', 'pids', self.ns_name], stdout=subprocess.PIPE)
        if result.stdout == b'':
            error = "RIFT process is not running in namespace {}".format(self.ns_name)
            self.report_check_result(step, False, error)
            return False
        self.report_check_result(step)
        return True 

def equalimages(imfile1, imfile2):
    """Returns True if two image files have equal content, False if not"""
    # If sizes differ, the images are not equal
    if shape(imfile1) != shape(imfile2):
        return False
    # Run imagemagick comparison commmand
    # This command returns with 0 if images are equal, 1 if they are not, 2 in case of error
    command = "compare -metric rmse %s %s null:" % (imfile1, imfile2)
    result = run(command, shell=True)
    if result.returncode == 2:
        raise IOError("Error while calling imagemagick compare method")
    return result.returncode == 0 

def compile_program(self, address_or_refid, file_path, is_module, script_args):
        address = self.parse_address_or_refid(address_or_refid)
        dependencies_file = self.handle_dependencies(file_path, is_module)
        if is_module:
            module_flag = " -m"
        else:
            module_flag = ""
        args = "cargo run -p compiler -- {} -a {}{}".format(
            file_path,
            address,
            module_flag
        )
        if dependencies_file:
            args += f" --deps={dependencies_file.name}"
        subprocess.run(args.split(), cwd="../libra/", check=True)
        if dependencies_file:
            dependencies_file.close()
        return file_path 

def sha3_256_mod():
    if has_sha3():
        return hashlib.sha3_256
    else:
        try:
            import sha3
        except ModuleNotFoundError:
            cmd = "python3 -m pip install --user pysha3"
            print("try to install pysha3 with following command:")
            print(cmd)
            subprocess.run(cmd.split(), check=True)
            import sha3
        return sha3.sha3_256 

def run_error(self, cmd, returncode, stdin=None, stderr=STDOUT, workdir=None):
        try:
            output = self.run(cmd, stdin=stdin, stderr=stderr, workdir=workdir)
        except CalledProcessError as e:
            if e.returncode != returncode:
                raise ValueError("Expected exit code {} but saw {}".format(returncode, e.returncode))
            else:
                output = e.stdout

        return output 

def run(self, cmd, stdin=None, stderr=STDOUT, workdir=None):
        p = run(cmd, check=True, encoding="utf8", cwd=workdir,
                input=stdin, stdout=PIPE, stderr=stderr)

        return p.stdout 

def all_namespaces():
    try:
        result = subprocess.run(['ip', 'netns', 'list'], stdout=subprocess.PIPE)
    except FileNotFoundError:
        fatal_error('"ip" command not found')
    output = result.stdout.decode('ascii')
    ns_list_with_ids = output.splitlines()
    ns_list = [ns.split()[0] for ns in  ns_list_with_ids]
    return ns_list 

def scan(cls, addr, addr_type):
        subprocess.run([cls.test_poc, 'BD_ADDR='+addr]) 

def _evaluate(self, x, out, *args, **kwargs):
        inputs = os.linesep.join([str(e) for e in x])
        res = subprocess.run(self.exec, input=inputs, text=True, stdout=subprocess.PIPE)
        val = np.array([float(e) for e in res.stdout.split()])
        out["F"] = val[0]
        out["G"] = val[1:] 

def run(file):
        result = subprocess.run(file.split(" "), stdout=subprocess.PIPE)
        print(result.stdout.decode('utf-8').rstrip("\n\r")) 

def run_script(self, script_folder, script_name, clean=None):
        if clean is not None and os.path.exists(clean):
            shutil.rmtree(clean)
        shutil.copy(
            os.path.join(EXAMPLES_DIR, script_folder, script_name),
            script_name
        )
        return subprocess.run(octave_call(script_name), shell=True) 

def mergealpha(rgbfile, alphafile, resfile):
    """Applies an alpha channel image to an RGB image"""
    if shape(rgbfile) != shape(alphafile):
        raise ValueError("Cant merge RGB and alpha images of differing sizes: %s vs %s" %
                         (str(shape(rgbfile)), str(shape(alphafile))))
    command = "convert %s %s -compose CopyOpacity -composite %s" % (rgbfile, alphafile, resfile)
    run(command, shell=True, check=True) 

def extractalpha(imfile, rgbfile, alphafile):
    """Decomposes an image file into the RGB channels and the alpha channel, saving both as separate image files"""
    if ismultilayer(imfile):
        raise ValueError("Cannot operate with multilayer images")
    # Alpha channel extraction
    command = "convert -alpha extract %s %s" % (imfile, alphafile)
    run(command, shell=True, check=True)
    # RGB channels extraction
    command = "convert -alpha off %s %s" % (imfile, rgbfile)
    run(command, shell=True, check=True) 

def composite(imfiles, outname):
    """Blends several image files together"""
    command = "composite"
    for imfile in imfiles:
        command += " " + imfile
    command += " " + outname
    run(command, shell=True, check=True) 

def feather(imfile, outname):
    """Produces a feathered version of an image. Note the output format must allow for an alpha channel"""
    command = 'convert %s -alpha set -virtual-pixel transparent -channel A -morphology Distance Euclidean:1,50\! ' \
              '+channel %s' % (imfile, outname)
    run(command, shell=True, check=True) 

def choptiles(imfile, xtiles, ytiles, overlap, outname):
    """Chops an image file into a geometry of overlapping tiles. Returns ordered list of generated tiles image files"""
    command = 'convert %s -crop %dx%d+%d+%d@ +repage +adjoin %s_%%d.png' % (
                imfile, xtiles, ytiles, overlap, overlap, outname
    )
    run(command, shell=True, check=True)
    return sorted(glob(outname + "_*.png"), key=lambda x: int(filename(x).split("_")[-1])) 

def resize(imfile, newsize):
    """Resize an image file to a new size.

    If a single value for newsize is provided, the image is rescaled to that size while keeping proportion.
    If an tuple/list with two values are given, the proportions of the image are changed to meet them.
    """
    if isinstance(newsize, int):
        command = "convert " + imfile + " -resize " + str(newsize) + " " + imfile
    else:
        command = "convert " + imfile + " -resize " + str(newsize[0]) + "x" + str(newsize[1]) + "! " + imfile
    run(command, shell=True) 

def shape(imfile):
    """Returns the shape of an image file"""
    result = run("convert " + imfile + ' -format "%w %h" info:', shell=True, check=True, stdout=PIPE)
    return [int(x) for x in result.stdout.decode("utf-8").split(" ")]