Python subprocess.check_call() Examples

def run_command(command, args=[], cwd=None, env=None, name='command'):
    def cmd_args_to_str(cmd_args):
        return ' '.join([arg if not ' ' in arg else '"%s"' % arg for arg in cmd_args])

    assert isinstance(command, str) and isinstance(args, list)
    args = [command] + args

    check_call_args = {}
    if cwd is not None:
        check_call_args['cwd'] = cwd
    if env is not None:
        check_call_args['env'] = env

    import subprocess
    try:
        print('Running command \'%s\': %s' % (name, subprocess.list2cmdline(args)))
        subprocess.check_call(args, **check_call_args)
        print('Command \'%s\' completed successfully' % name)
    except subprocess.CalledProcessError as e:
        raise BuildError('\'%s\' exited with error code: %s' % (name, e.returncode)) 

def ensure_session_manager_plugin():
    session_manager_dir = os.path.join(config.user_config_dir, "bin")
    PATH = os.environ.get("PATH", "") + ":" + session_manager_dir
    if shutil.which("session-manager-plugin", path=PATH):
        subprocess.check_call(["session-manager-plugin"], env=dict(os.environ, PATH=PATH))
    else:
        os.makedirs(session_manager_dir, exist_ok=True)
        target_path = os.path.join(session_manager_dir, "session-manager-plugin")
        if platform.system() == "Darwin":
            download_session_manager_plugin_macos(target_path=target_path)
        elif platform.linux_distribution()[0] == "Ubuntu":
            download_session_manager_plugin_linux(target_path=target_path)
        else:
            download_session_manager_plugin_linux(target_path=target_path, pkg_format="rpm")
        os.chmod(target_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
        subprocess.check_call(["session-manager-plugin"], env=dict(os.environ, PATH=PATH))
    return shutil.which("session-manager-plugin", path=PATH) 

def mpi_fork(n, extra_mpi_args=[]):
    """Re-launches the current script with workers
    Returns "parent" for original parent, "child" for MPI children
    """
    if n <= 1:
        return "child"
    if os.getenv("IN_MPI") is None:
        env = os.environ.copy()
        env.update(
            MKL_NUM_THREADS="1",
            OMP_NUM_THREADS="1",
            IN_MPI="1"
        )
        # "-bind-to core" is crucial for good performance
        args = ["mpirun", "-np", str(n)] + \
            extra_mpi_args + \
            [sys.executable]

        args += sys.argv
        subprocess.check_call(args, env=env)
        return "parent"
    else:
        install_mpi_excepthook()
        return "child" 

def sync_upload(rootfs, src, dest):
    fs = Filesystem(rootfs)
    if not fs.implemented():
        return
    print("Adding current directory to the filesystem..")
    with tempfile.TemporaryDirectory() as tmpdirname:
        files = [i for i in os.listdir(".") if i != "arm_now" and not i.startswith("-")]
        if files:
            tar = tmpdirname + "/current_directory.tar"
            subprocess.check_call(["tar", "cf", tar] + files)
            subprocess.check_call("e2cp -G 0 -O 0".split(' ') + [tar, rootfs + ":/"])
            fs.create("/etc/init.d/S95_sync_current_diretory", """
                        cd {dest}
                        tar xf /current_directory.tar
                        rm /current_directory.tar
                        rm /etc/init.d/S95_sync_current_diretory
                        """.format(dest=dest), right=555)

    # TODO: check rootfs fs against parameter injection
    fs.create("/sbin/save", """
                cd {dest}
                tar cf /root.tar *
                sync
                """.format(dest=dest), right=555) 

def run_ida(db, is_64_bit, timeout, script, *args):
    if os.path.exists(os.path.join(IDA_DIR, "idat")):
        # This is IDA >= 7.0
        IDA_EXECUTABLE = os.path.join(IDA_DIR, "idat")
    else:
        IDA_EXECUTABLE = os.path.join(IDA_DIR, "idal")

    if is_64_bit:
        ida = "{}64".format(IDA_EXECUTABLE)
    else:
        ida = IDA_EXECUTABLE

    cmd = (ida, "-S{} {}".format(script, " ".join("\"{}\"".format(x) for x in args)), "-B", db)
    env = os.environ.copy()
    env["TVHEADLESS"] = "true"
    env["IDALOG"] = os.path.join(LOGS_DIR, datetime.datetime.strftime(datetime.datetime.now(), "ida_%Y-%m-%d_%H-%M-%S.%f.log"))
    logger.info("Executing command %s, log output is in '%s'", " ".join("'%s'" % x for x in cmd), env["IDALOG"])
    try:
        check_call(cmd, timeout = timeout, env = env)
    except OSError as err:
        if err.errno == -9:
            raise TimeoutError(err.errno, "Program execution timed out")
        else:
            raise err 

def gpt_device(self, dev_name):
        disk_dev = self.physical_disk(dev_name)
        cmd = ['parted', disk_dev, '-s', 'print']
        with open(os.devnull) as devnull:
            p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE, stdin=devnull)
            _cmd_out, _err_out = p.communicate()
            p.wait()
        if p.returncode != 0:
            lang = os.getenv('LANG')
            encoding = lang.rsplit('.')[-1] if lang else 'utf-8'
            raise RuntimeError(str(_err_out, encoding))
        subprocess.check_call(['partprobe', disk_dev])
        if b'msdos' in _cmd_out:
            return False
        if b'gpt' in _cmd_out:
            return True
        raise RuntimeError("Disk '%s' is uninitialized and not usable." %
                           disk_dev) 

def delete_local_docker_cache(docker_tag):
    """
    Delete the local docker cache for the entire docker image chain
    :param docker_tag: Docker tag
    :return: None
    """
    history_cmd = ['docker', 'history', '-q', docker_tag]

    try:
        image_ids_b = subprocess.check_output(history_cmd)
        image_ids_str = image_ids_b.decode('utf-8').strip()
        layer_ids = [id.strip() for id in image_ids_str.split('\n') if id != '<missing>']

        delete_cmd = ['docker', 'image', 'rm', '--force']
        delete_cmd.extend(layer_ids)
        subprocess.check_call(delete_cmd)
    except subprocess.CalledProcessError as error:
        # Could be caused by the image not being present
        logging.debug('Error during local cache deletion %s', error) 

def main(_):
  shard_urls = fetch.get_urls_for_shard(FLAGS.urls_dir, FLAGS.shard_id)
  num_groups = int(math.ceil(len(shard_urls) / fetch.URLS_PER_CLIENT))
  tf.logging.info("Launching get_references_web_single_group sequentially for "
                  "%d groups in shard %d. Total URLs: %d",
                  num_groups, FLAGS.shard_id, len(shard_urls))
  command_prefix = FLAGS.command.split() + [
      "--urls_dir=%s" % FLAGS.urls_dir,
      "--shard_id=%d" % FLAGS.shard_id,
      "--debug_num_urls=%d" % FLAGS.debug_num_urls,
  ]
  with utils.timing("all_groups_fetch"):
    for i in range(num_groups):
      command = list(command_prefix)
      out_dir = os.path.join(FLAGS.out_dir, "process_%d" % i)
      command.append("--out_dir=%s" % out_dir)
      command.append("--group_id=%d" % i)
      try:
        # Even on 1 CPU, each group should finish within an hour.
        sp.check_call(command, timeout=60*60)
      except sp.TimeoutExpired:
        tf.logging.error("Group %d timed out", i) 

def mpi_fork(n, bind_to_core=False):
    """Re-launches the current script with workers
    Returns "parent" for original parent, "child" for MPI children
    """
    if n<=1: 
        return "child"
    if os.getenv("IN_MPI") is None:
        env = os.environ.copy()
        env.update(
            MKL_NUM_THREADS="1",
            OMP_NUM_THREADS="1",
            IN_MPI="1"
        )
        args = ["mpirun", "-np", str(n)]
        if bind_to_core:
            args += ["-bind-to", "core"]
        args += [sys.executable] + sys.argv
        subprocess.check_call(args, env=env)
        return "parent"
    else:
        return "child" 

def update():
    """Update to the latest pages."""
    repo_directory = get_config()['repo_directory']
    os.chdir(repo_directory)
    click.echo("Check for updates...")

    local = subprocess.check_output('git rev-parse master'.split()).strip()
    remote = subprocess.check_output(
        'git ls-remote https://github.com/tldr-pages/tldr/ HEAD'.split()
    ).split()[0]
    if local != remote:
        click.echo("Updating...")
        subprocess.check_call('git checkout master'.split())
        subprocess.check_call('git pull --rebase'.split())
        build_index()
        click.echo("Update to the latest and rebuild the index.")
    else:
        click.echo("No need for updates.") 

def mpi_fork(n, bind_to_core=False):
    """Re-launches the current script with workers
    Returns "parent" for original parent, "child" for MPI children
    """
    if n<=1: 
        return "child"
    if os.getenv("IN_MPI") is None:
        env = os.environ.copy()
        env.update(
            MKL_NUM_THREADS="1",
            OMP_NUM_THREADS="1",
            IN_MPI="1"
        )
        args = ["mpirun", "-np", str(n)]
        if bind_to_core:
            args += ["-bind-to", "core"]
        args += [sys.executable] + sys.argv
        subprocess.check_call(args, env=env)
        return "parent"
    else:
        return "child" 

def draw_computational_graph(outputs, filepath):
    """Draw a computational graph and write to a given file.

    Args:
        outputs (object): Output(s) of the computational graph. It must be
            a Variable, an ActionValue, a Distribution or a list of them.
        filepath (str): Filepath to write a graph without file extention.
            A DOT file will be saved with ".gv" extension added.
            If Graphviz's dot command is available, a PNG file will also be
            saved with ".png" extension added.
    """
    variables = collect_variables(outputs)
    g = chainer.computational_graph.build_computational_graph(variables)
    gv_filepath = filepath + '.gv'
    with open(gv_filepath, 'w') as f:
        f.write(g.dump())
    if is_graphviz_available():
        png_filepath = filepath + '.png'
        subprocess.check_call(
            ['dot', '-Tpng', gv_filepath, '-o', png_filepath]) 

def mount_all(self):
        """Mount all attached USB drives.  Readonly is a boolean that specifies
        if the drives should be mounted read only (defaults to true).
        """
        self.remove_all()
        # Enumerate USB drive partitions by path like /dev/sda1, etc.
        nodes = [x.device_node for x in self._context.list_devices(subsystem='block', DEVTYPE='partition')
                 if 'ID_BUS' in x and x['ID_BUS'] == 'usb']
        # Mount each drive under the mount root.
        for i, node in enumerate(nodes):
            path = self._root + str(i)
            subprocess.call(['mkdir', path])
            args = ['mount']
            if self._readonly:
                args.append('-r')
            args.extend([node, path])
            subprocess.check_call(args)

        return nodes 

def __bleu_score__(self, directory, val=True):
        '''
        Loss is only weakly correlated with improvements in BLEU,
        and thus improvements in human judgements. Let's also track
        BLEU score of a subset of generated sentences in the val split
        to decide on early stopping, etc.
        '''

        prefix = "val" if val else "test"

        self.extract_references(directory, split=prefix)

        subprocess.check_call(
            ['perl multi-bleu.perl %s/%s_reference.ref < %s/%sGenerated > %s/%sBLEU'
             % (directory, prefix, directory, prefix, directory, prefix)],
            shell=True)
        bleudata = open("%s/%sBLEU" % (directory, prefix)).readline()
        data = bleudata.split(",")[0]
        bleuscore = data.split("=")[1]
        bleu = float(bleuscore.lstrip())
        return bleu 

def bleu_score(self, directory, val=True):
        '''
        PPLX is only weakly correlated with improvements in BLEU,
        and thus improvements in human judgements. Let's also track
        BLEU score of a subset of generated sentences in the val split
        to decide on early stopping, etc.
        '''

        prefix = "val" if val else "test"
        self.extract_references(directory, val)

        subprocess.check_call(
            ['perl multi-bleu.perl %s/%s_reference.ref < %s/%sGenerated | tee %s/%sBLEU'
             % (directory, prefix, directory, prefix, directory, prefix)], shell=True)
        bleudata = open("%s/%sBLEU" % (directory, prefix)).readline()
        data = bleudata.split(",")[0]
        bleuscore = data.split("=")[1]
        bleu = float(bleuscore.lstrip())
        return bleu 

def main():
    """Run the scalability exercise.

    Define the model, a list with different number of threads and a maximum number of
    function evaluations.

    """
    model = "kw_97_basic"
    maxfun = 3

    filepath = Path(__file__).resolve().parent / "run_single_scalability_exercise.py"

    # Run Python
    for n_threads in [2, 4, 6, 8, 10, 12, 14]:
        subprocess.check_call(
            ["python", str(filepath), model, str(maxfun), str(n_threads)]
        ) 

def ensureIndexed(bedPath, preset="bed", trySorting=True):
    if not bedPath.endswith(".gz"):
        if not os.path.exists(bedPath+".gz"):
            logging.info("bgzf compressing {}".format(bedPath))
            pysam.tabix_compress(bedPath, bedPath+".gz")
            if not os.path.exists(bedPath+".gz"):
                raise Exception("Failed to create compress {preset} file for {file}; make sure the {preset} file is "
                    "sorted and the directory is writeable".format(preset=preset, file=bedPath))
        bedPath += ".gz"
    if not os.path.exists(bedPath+".tbi"):
        logging.info("creating tabix index for {}".format(bedPath))
        pysam.tabix_index(bedPath, preset=preset)
        if not os.path.exists(bedPath+".tbi"):
            raise Exception("Failed to create tabix index file for {file}; make sure the {preset} file is "
                "sorted and the directory is writeable".format(preset=preset, file=bedPath))

    line = next(pysam.Tabixfile(bedPath).fetch())
    if len(line.strip().split("\t")) < 6 and preset == "bed":
        raise AnnotationError("BED files need to have at least 6 (tab-delimited) fields (including "
            "chrom, start, end, name, score, strand; score is unused)")
    if len(line.strip().split("\t")) < 9 and preset == "gff":
        raise AnnotationError("GFF/GTF files need to have at least 9 tab-delimited fields")

    return bedPath


# def sortFile(uncompressedPath, preset):
#     if preset == "bed":
#         fields = {"chrom":0, "start":1, "end":2}
#     elif preset == "gff":
#         fields = {"chrom":0, "start":3, "end":4}

#     sortCommand = "sort -k{chrom}V -k{start}n -k{end}n".format(**fields)

#     tabixCommand = "{sort} {path} | bgzip > {path}.gz".format(sort=sortCommand, path=uncompressedPath)

#     logging.info("Trying to sort input annotation file with command:")
#     logging.info("  {}".format(tabixCommand))

#     subprocess.check_call(tabixCommand, shell=True) 

def checkWebkitToPDF():
    try:
        subprocess.check_call("webkitToPDF", stderr=subprocess.PIPE, shell=True)
        return True
    except subprocess.CalledProcessError:
        return False 

def checkRSVGConvert():
    try:
        subprocess.check_call("rsvg-convert -v", stdout=subprocess.PIPE, shell=True)
        return True
    except subprocess.CalledProcessError:
        return False 

def checkInkscape():
    try:
        subprocess.check_call("inkscape --version", stdout=subprocess.PIPE, shell=True)
        return True
    except subprocess.CalledProcessError:
        return False 

def _convertSVG_webkitToPDF(inpath, outpath, outformat):
    if outformat.lower() != "pdf":
        return None

    try:
        cmd = "webkitToPDF {} {}".format(inpath, outpath)
        subprocess.check_call(cmd, shell=True)#, stderr=subprocess.PIPE)
    except subprocess.CalledProcessError:
        return None

    return open(outpath, "rb").read() 

def _convertSVG_rsvg_convert(inpath, outpath, outformat):
    options = ""
    outformat = outformat.lower()
    if outformat == "png":
        options = "-a --background-color white"

    try:
        subprocess.check_call("rsvg-convert -f {} {} -o {} {}".format(outformat, options, outpath, inpath), shell=True)
    except subprocess.CalledProcessError as e:
        print("EXPORT ERROR:", str(e))

    return open(outpath, "rb").read()


# def test():
#     base = """  <svg><rect x="10" y="10" height="100" width="100" style="stroke:#ffff00; stroke-width:3; fill: #0000ff"/><text x="25" y="25" fill="blue">{}</text></svg>"""
#     svgs = [base.format("track {}".format(i)) for i in range(5)]

#     tc = TrackCompositor(200, 600)
#     for i, svg in enumerate(svgs):
#         tc.addTrack(svg, i, viewbox="0 0 110 110")

#     outf = open("temp.svg", "w")
#     outf.write(tc.render())
#     outf.flush()
#     outf.close()

#     pdfPath = convertSVGToPDF("temp.svg")
#     subprocess.check_call("open {}".format(pdfPath), shell=True)

# if __name__ == '__main__':
#     test()

#     import sys
#     print(canConvertSVGToPDF(), file=sys.stderr) 

def attach(args):
    if args.instance is None:
        args.instance = get_metadata("instance-id")
    devices = args.device if args.device else ["xvd" + chr(i + 1) for i in reversed(range(ord("a"), ord("z")))]
    for i, device in enumerate(devices):
        try:
            args.device = devices[i]
            res = attach_volume(args)
            break
        except ClientError as e:
            if re.search("VolumeInUse.+already attached to an instance", str(e)):
                if resources.ec2.Volume(args.volume_id).attachments[0]["InstanceId"] == args.instance:
                    logger.warn("Volume %s is already attached to instance %s", args.volume_id, args.instance)
                    break
            if i + 1 < len(devices) and re.search("InvalidParameterValue.+Attachment point.+is already in use", str(e)):
                logger.warn("BDM node %s is already in use, looking for next available node", devices[i])
                continue
            raise
    res = clients.ec2.get_waiter("volume_in_use").wait(VolumeIds=[args.volume_id])
    if args.format or args.mount:
        for i in range(30):
            try:
                find_devnode(args.volume_id)
                break
            except Exception:
                logger.debug("Waiting for device node to appear for %s", args.volume_id)
                time.sleep(1)
    if args.format:
        logger.info("Formatting %s (%s)", args.volume_id, find_devnode(args.volume_id))
        label = get_fs_label(args.volume_id)
        command = get_mkfs_command(fs_type=args.format, label=label) + find_devnode(args.volume_id)
        subprocess.check_call(command, shell=True, stdout=sys.stderr.buffer)
    if args.mount:
        logger.info("Mounting %s at %s", args.volume_id, args.mount)
        subprocess.check_call(["mount", find_devnode(args.volume_id), args.mount], stdout=sys.stderr.buffer)
    return res 

def download_session_manager_plugin_linux(target_path, pkg_format="deb"):
    assert pkg_format in {"deb", "rpm"}
    if pkg_format == "deb":
        sm_plugin_key = "plugin/latest/ubuntu_64bit/session-manager-plugin.deb"
    else:
        sm_plugin_key = "plugin/latest/linux_64bit/session-manager-plugin.rpm"
    with tempfile.TemporaryDirectory() as td:
        sm_archive_path = os.path.join(td, os.path.basename(sm_plugin_key))
        clients.s3.download_file(sm_plugin_bucket, sm_plugin_key, sm_archive_path)
        if pkg_format == "deb":
            subprocess.check_call(["dpkg", "-x", sm_archive_path, td])
        elif pkg_format == "rpm":
            command = "rpm2cpio '{}' | cpio --extract --make-directories --directory '{}'"
            subprocess.check_call(command.format(sm_archive_path, td), shell=True)
        shutil.move(os.path.join(td, "usr/local/sessionmanagerplugin/bin/session-manager-plugin"), target_path) 

def sync_download(rootfs, src, dest):
    fs = Filesystem(rootfs)
    if not fs.implemented():
        return
    fs.get(src, dest)
    if os.path.exists("root.tar"):
        subprocess.check_call("tar xf root.tar".split(' '))
        os.unlink("root.tar")
    else:
        pgreen("Use the 'save' command before exiting the vm to retrieve all files on the host") 

def put(self, src, dest, right=444):
        subprocess.check_call("e2cp -G 0 -O 0 -P".split(' ') + [str(right), src, self.rootfs + ":" + dest]) 

def get(self, src, dest):
        subprocess.check_call(["e2cp", self.rootfs + ":" + src, dest]) 

def rm(self, filename):
        def e2rm_warning(_exception):
            porange("WARNING: e2rm file already suppressed")
        with exall(subprocess.check_call, subprocess.CalledProcessError, e2rm_warning):
            subprocess.check_call(["e2rm", self.rootfs + ":" + filename]) 

def sed(self, regex, path, right=444):
        """ Replace with sed in the roofs
        Example: fs.sed('s/init.d\/S/init.d\/K/g', '/etc/init.d/rcK', right=755)
        Insecure !! command injection here but regex is not exposed to user input
        """
        with tempfile.TemporaryDirectory() as tempdir:
            print("Tempdir {}".format(tempdir))
            new = tempdir + "/new"
            old = tempdir + "/old"
            self.get(path, old)
            subprocess.check_call("sed '{regex}' {old} > {new}".format(
                regex=regex, new=new, old=old), shell=True)
            self.put(new, path, right=right) 

def resize(self, size):
        subprocess.check_call(["qemu-img", "resize", self.rootfs, size])
        subprocess.check_call(["e2fsck", "-fy", self.rootfs])
        subprocess.check_call(["resize2fs", self.rootfs])
        subprocess.check_call(["ls", "-lh", self.rootfs])
        pgreen("[+] Resized to {size}".format(size=size))