Python subprocess.call() Examples

def call(cls, cmd, shell=False):
        '''
            Run command with arguments, wait to complete and return ``True`` on success.

        :param cls: The class as implicit first argument.
        :param cmd: Command string to be executed.
        :returns  : ``True`` on success, otherwise ``None``.
        :rtype    : ``bool``
        '''
        logger = logging.getLogger('SPOT.INGEST.COMMON.UTIL')
        logger.debug('Execute command: {0}'.format(cmd))

        try:
            subprocess.call(cmd, shell=shell)
            return True

        except Exception as exc:
            logger.error('[{0}] {1}'.format(exc.__class__.__name__, exc.message)) 

def main(vfl, run=False, version=True, threads=1, time=True):
    """Compiles and optionally runs vex code. Arguments:
            vfl:     the path to compile '/some/code.vlf'
            run:     if this is true, we call execute() and thus vexexec
            version: print the houdini version called for compiling and running
            threads: number of threads when run is true
            time:    print the execution time when running
    """
    hfs, version = find_houdini()
    if version:
        print('Using houdini path:\n\t%s' % hfs)

    # compile
    print('Compiling:\n\t%s' % vfl)
    bin = os.path.join(hfs, 'bin')
    vex = build(bin, vfl)

    # execute
    if run:
        print('Executing:\n\t%s' % vex)
        execute(bin, vex, threads, time) 

def install_update():
    """If a newer release is available, download and install it.

    :returns: ``True`` if an update is installed, else ``False``

    """
    update_data = wf().cached_data('__workflow_update_status', max_age=0)

    if not update_data or not update_data.get('available'):
        wf().logger.info('no update available')
        return False

    local_file = download_workflow(update_data['download_url'])

    wf().logger.info('installing updated workflow ...')
    subprocess.call(['open', local_file])

    update_data['available'] = False
    wf().cache_data('__workflow_update_status', update_data)
    return True 

def convert_image(inpath, outpath, size):
    """Convert an image file using ``sips``.

    Args:
        inpath (str): Path of source file.
        outpath (str): Path to destination file.
        size (int): Width and height of destination image in pixels.

    Raises:
        RuntimeError: Raised if ``sips`` exits with non-zero status.
    """
    cmd = [
        b'sips',
        b'-z', str(size), str(size),
        inpath,
        b'--out', outpath]
    # log().debug(cmd)
    with open(os.devnull, 'w') as pipe:
        retcode = subprocess.call(cmd, stdout=pipe, stderr=subprocess.STDOUT)

    if retcode != 0:
        raise RuntimeError('sips exited with %d' % retcode) 

def main(args):
  print_in_box('Validating submission ' + args.submission_filename)
  random.seed()
  temp_dir = args.temp_dir
  delete_temp_dir = False
  if not temp_dir:
    temp_dir = tempfile.mkdtemp()
    logging.info('Created temporary directory: %s', temp_dir)
    delete_temp_dir = True
  validator = validate_submission_lib.SubmissionValidator(temp_dir,
                                                          args.use_gpu)
  if validator.validate_submission(args.submission_filename,
                                   args.submission_type):
    print_in_box('Submission is VALID!')
  else:
    print_in_box('Submission is INVALID, see log messages for details')
  if delete_temp_dir:
    logging.info('Deleting temporary directory: %s', temp_dir)
    subprocess.call(['rm', '-rf', temp_dir]) 

def shell_call(command, **kwargs):
  """Calls shell command with parameter substitution.

  Args:
    command: command to run as a list of tokens
    **kwargs: dirctionary with substitutions

  Returns:
    whether command was successful, i.e. returned 0 status code

  Example of usage:
    shell_call(['cp', '${A}', '${B}'], A='src_file', B='dst_file')
  will call shell command:
    cp src_file dst_file
  """
  command = list(command)
  for i in range(len(command)):
    m = CMD_VARIABLE_RE.match(command[i])
    if m:
      var_id = m.group(1)
      if var_id in kwargs:
        command[i] = kwargs[var_id]
  return subprocess.call(command) == 0 

def run(self, input_dir, output_dir, epsilon):
    """Runs attack inside Docker.

    Args:
      input_dir: directory with input (dataset).
      output_dir: directory where output (adversarial images) should be written.
      epsilon: maximum allowed size of adversarial perturbation,
        should be in range [0, 255].
    """
    print('Running attack ', self.name)
    cmd = [self.docker_binary(), 'run',
           '-v', '{0}:/input_images'.format(input_dir),
           '-v', '{0}:/output_images'.format(output_dir),
           '-v', '{0}:/code'.format(self.directory),
           '-w', '/code',
           self.container,
           './' + self.entry_point,
           '/input_images',
           '/output_images',
           str(epsilon)]
    print(' '.join(cmd))
    subprocess.call(cmd) 

def run(self, input_dir, output_dir):
    """Runs defense inside Docker.

    Args:
      input_dir: directory with input (adversarial images).
      output_dir: directory to write output (classification result).
    """
    print('Running defense ', self.name)
    cmd = [self.docker_binary(), 'run',
           '-v', '{0}:/input_images'.format(input_dir),
           '-v', '{0}:/output_data'.format(output_dir),
           '-v', '{0}:/code'.format(self.directory),
           '-w', '/code',
           self.container,
           './' + self.entry_point,
           '/input_images',
           '/output_data/result.csv']
    print(' '.join(cmd))
    subprocess.call(cmd) 

def save_id_to_path_mapping(self):
    """Saves mapping from submission IDs to original filenames.

    This mapping is saved as CSV file into target directory.
    """
    if not self.id_to_path_mapping:
      return
    with open(self.local_id_to_path_mapping_file, 'w') as f:
      writer = csv.writer(f)
      writer.writerow(['id', 'path'])
      for k, v in sorted(iteritems(self.id_to_path_mapping)):
        writer.writerow([k, v])
    cmd = ['gsutil', 'cp', self.local_id_to_path_mapping_file,
           os.path.join(self.target_dir, 'id_to_path_mapping.csv')]
    if subprocess.call(cmd) != 0:
      logging.error('Can\'t copy id_to_path_mapping.csv to target directory') 

def shell_call(command, **kwargs):
  """Calls shell command with parameter substitution.

  Args:
    command: command to run as a list of tokens
    **kwargs: dirctionary with substitutions

  Returns:
    whether command was successful, i.e. returned 0 status code

  Example of usage:
    shell_call(['cp', '${A}', '${B}'], A='src_file', B='dst_file')
  will call shell command:
    cp src_file dst_file
  """
  command = list(command)
  for i in range(len(command)):
    m = CMD_VARIABLE_RE.match(command[i])
    if m:
      var_id = m.group(1)
      if var_id in kwargs:
        command[i] = kwargs[var_id]
  return subprocess.call(command) == 0 

def run_without_time_limit(self, cmd):
    """Runs docker command without time limit.

    Args:
      cmd: list with the command line arguments which are passed to docker
        binary

    Returns:
      how long it took to run submission in seconds

    Raises:
      WorkerError: if error occurred during execution of the submission
    """
    cmd = [DOCKER_BINARY, 'run', DOCKER_NVIDIA_RUNTIME] + cmd
    logging.info('Docker command: %s', ' '.join(cmd))
    start_time = time.time()
    retval = subprocess.call(cmd)
    elapsed_time_sec = long(time.time() - start_time)
    logging.info('Elapsed time of attack: %d', elapsed_time_sec)
    logging.info('Docker retval: %d', retval)
    if retval != 0:
      logging.warning('Docker returned non-zero retval: %d', retval)
      raise WorkerError('Docker returned non-zero retval ' + str(retval))
    return elapsed_time_sec 

def fetch_attacks_data(self):
    """Initializes data necessary to execute attacks.

    This method could be called multiple times, only first call does
    initialization, subsequent calls are noop.
    """
    if self.attacks_data_initialized:
      return
    # init data from datastore
    self.submissions.init_from_datastore()
    self.dataset_batches.init_from_datastore()
    self.adv_batches.init_from_datastore()
    # copy dataset locally
    if not os.path.exists(LOCAL_DATASET_DIR):
      os.makedirs(LOCAL_DATASET_DIR)
    eval_lib.download_dataset(self.storage_client, self.dataset_batches,
                              LOCAL_DATASET_DIR,
                              os.path.join(LOCAL_DATASET_COPY,
                                           self.dataset_name, 'images'))
    # download dataset metadata
    self.read_dataset_metadata()
    # mark as initialized
    self.attacks_data_initialized = True 

def copy_iso(src, dst):
    """
    A simple wrapper for copying larger files. This is necessary as
    shutil copy files is much slower under Windows platform
    :param src: Path to source file
    :param dst: Destination directory
    :return:
    """
    if platform.system() == "Windows":
        # Note that xcopy asks if the target is a file or a directory when
        # source filename (or dest filename) contains space(s) and the target
        # does not exist.
        assert os.path.exists(dst)
        subprocess.call(['xcopy', '/Y', src, dst], shell=True)
    elif platform.system() == "Linux":
        shutil.copy(src, dst) 

def find_qemu():
        """
        Check if QEMU is available on host system and return path of the binary
        :return: path to QEMU program or None otherwise.
        """
        if platform.system() == "Linux":
            if subprocess.call('which qemu-system-x86_64', shell=True) == 0:
                qemu = "qemu-system-x86_64"
            elif subprocess.call('which qemu', shell=True) == 0:
                qemu = "qemu"
            else:
                qemu = ""

        elif platform.system() == "Windows":
            qemu = find_qemu_exe()

        if qemu:
            log("QEMU: using " + qemu)
        else:
            log("QEMU: ERROR: not found!")

        return qemu 

def install_dependency_package():
        if subprocess.call("which pacman", shell=True) == 0:
            subprocess.call("pacman -Sy --noconfirm", shell=True)
            # Thank you Neitsab for "--needed"  argument.
            if subprocess.call("pacman -S --needed --noconfirm p7zip python-pyqt5 mtools python3-six parted util-linux python-dbus") == 0:
                result = True
        elif subprocess.call("which yum", shell=True) == 0:
            subprocess.call("yum check-update", shell=True)
            if subprocess.call("dnf install mtools python3-PyQt5 util-linux python3-six parted p7zip p7zip-plugins python3-pyudev python3-dbus -y", shell=True) == 0:
                result = True
        elif subprocess.call("which apt-get", shell=True) == 0:
            subprocess.call("apt-get -q update", shell=True)
            if subprocess.call("apt-get -q -y install python3-pyqt5 p7zip-full parted util-linux python3-pyudev mtools python3-dbus", shell=True) == 0:
                result = True
        elif subprocess.call("which zypper", shell=True) == 0:
            subprocess.call("zypper refresh", shell=True)
            if subprocess.call("zypper install -y mtools python3-qt5 p7zip python3-pyudev python3-six util-linux parted", shell=True) == 0:
                result = True
        elif subprocess.call("which urpmi", shell=True) == 0:
            subprocess.call("urpmi.update -a", shell=True)
            if subprocess.call("urpmi install -auto mtools util-linux p7zip python3-pyudev python3-six parted python3-qt5", shell=True) == 0:
                result = True

        return bool(result) 

def bamToGold(bamtogold, merged, out, metadata, threads):
    """
    Calls the bamToGold script for all of the merged bam files, creating the gold standard
    """
    out_name = os.path.join(out, "anonymous_gsa.fasta")
    all_files = os.listdir(merged)
    bams = []
    for f in all_files:
        if f.endswith(".bam"):
            bams.append(f)
    for bam in bams:
        genome = bam.rstrip(".bam")
        otu, ncbi, novelty, path = metadata[genome]
        cmd = "{bamToGold} -r {path} -b {bam} -l 1 -c 1 >> {gsa}".format(
            bamToGold = bamtogold,
            path = path,
            bam = os.path.join(out,"bam",bam),
            gsa = out_name
        )
        subprocess.call([cmd],shell=True) 

def merge_bam_files(bams_per_genome, out, threads):
    """
    Merges (+sort +index)  all given bam files per genome (exact paths, single sample/multiple runs or multiple samples)
    """
    out_path = os.path.join(out,"bam")
    os.mkdir(out_path)
    for genome in bams_per_genome:
        list_of_bam = " ".join(bams_per_genome[genome]) # can be used as input to samtools immediately
        header = fix_headers(genome, bams_per_genome[genome], out_path)
        if header is not None:
            for bam in bams_per_genome[genome]: # add new header to all bam files
                cmd = "samtools reheader {header} {bam} >> {out}/out.bam; mv {out}/out.bam {bam}".format(
                    header = header,
                    out = out_path,
                    bam = bam
                )
                subprocess.call([cmd],shell=True)
        cmd = "samtools merge -@ {threads} - {bam_files} | samtools sort -@ {threads} - {path}/{genome}; samtools index {path}/{genome}.bam".format(
            threads = threads,
            bam_files = list_of_bam,
            path = out_path,
            genome = genome
        )
        subprocess.call([cmd],shell=True) # this runs a single command at a time (but that one multi threaded)
    return out_path 

def run_test_trials(args):
    test_path = args.test_path + ":" + args.test_name
    logging.info("Testing: %s", test_path)

    new_env = os.environ.copy()
    new_env["MXNET_TEST_COUNT"] = str(args.num_trials)
    
    if args.seed is None:
        logging.info("No test seed provided, using random seed")
    else:
        new_env["MXNET_TEST_SEED"] = str(args.seed)

    verbosity = "--verbosity=" + str(args.verbosity)

    code = subprocess.call(["nosetests", verbosity, test_path], 
                           env = new_env)
    
    logging.info("Nosetests terminated with exit code %d", code) 

def launchFile(filepath):
    if sys.platform.startswith('darwin'):
        subprocess.call(('open', filepath))
    elif os.name == 'nt':
        os.startfile(filepath)
    elif os.name == 'posix':
        subprocess.call(('xdg-open', filepath))


############################ String utilities ############################ 

def remove_kafka_topic(cls,zk,topic,logger):
        rm_kafka_topic = "kafka-topics --delete --zookeeper {0} --topic {1}".format(zk,topic)
        try:
            logger.info("SPOT.Utils: Executing: {0}".format(rm_kafka_topic))
            subprocess.call(rm_kafka_topic,shell=True)

        except subprocess.CalledProcessError as e:
            logger.error("SPOT.Utils: There was an error executing: {0}".format(e.cmd))
            sys.exit(1) 

def creat_hdfs_folder(cls,hdfs_path,logger):
        hadoop_create_folder="hadoop fs -mkdir -p {0}".format(hdfs_path)
        logger.info("SPOT.Utils: Creating hdfs folder: {0}".format(hadoop_create_folder))
        subprocess.call(hadoop_create_folder,shell=True) 

def load_to_hdfs(cls,file_local_path,file_hdfs_path,logger):
        # move file to hdfs.
        load_to_hadoop_script = "hadoop fs -moveFromLocal {0} {1}".format(file_local_path,file_hdfs_path)
        logger.info("SPOT.Utils: Loading file to hdfs: {0}".format(load_to_hadoop_script))
        subprocess.call(load_to_hadoop_script,shell=True) 

def execute_cmd(cls,command,logger):

        try:
            logger.info("SPOT.Utils: Executing: {0}".format(command))
            subprocess.call(command,shell=True)

        except subprocess.CalledProcessError as e:
            logger.error("SPOT.Utils: There was an error executing: {0}".format(e.cmd))
            sys.exit(1) 

def load_to_hdfs(cls,file_local_path,file_hdfs_path,logger):
        # move file to hdfs.
        load_to_hadoop_script = "hadoop fs -moveFromLocal {0} {1}".format(file_local_path,file_hdfs_path)
        logger.info("SPOT.Utils: Loading file to hdfs: {0}".format(load_to_hadoop_script))
        subprocess.call(load_to_hadoop_script,shell=True) 

def execute_cmd(cls,command,logger):

        try:
            logger.info("SPOT.Utils: Executing: {0}".format(command))
            subprocess.call(command,shell=True)

        except subprocess.CalledProcessError as e:
            logger.error("SPOT.Utils: There was an error executing: {0}".format(e.cmd))
            sys.exit(1) 

def execute_hive_cmd(cls,command,logger):

        try:
            logger.info("SPOT.Utils: Executing hive command: hive -e \"{0}\"".format(command))
            subprocess.call("hive -e \"{0}\"".format(command),shell=True)

        except subprocess.CalledProcessError as e:
            logger.error("SPOT.Utils: There was an error executing: {0}".format(e.cmd))
            sys.exit(1) 

def get_ml_results_form_hdfs(cls, hdfs_file_path, local_path):

        # get results from hdfs.
        get_results_cmd = "hadoop fs -get {0} {1}/.".format(hdfs_file_path, local_path)
        subprocess.call(get_results_cmd, shell=True)
        return get_results_cmd 

def _has_stty_available(self):
        devnull = open(os.devnull, "w")

        try:
            exit_code = subprocess.call(["stty"], stdout=devnull, stderr=devnull)
        except Exception:
            exit_code = 2

        return exit_code == 0 

def has_tty_available():
    devnull = open(os.devnull, "w")
    exit_code = subprocess.call(["stty", "2"], stdout=devnull, stderr=devnull)

    return exit_code == 0 

def detach(args):
    """
    Detach an EBS volume from an EC2 instance.

    If *volume_id* does not start with "vol-", it is interpreted as a mountpoint on the local instance,
    mapped to its underlying EBS volume, unmounted and detached.
    """
    if args.volume_id.startswith("vol-"):
        volume_id = args.volume_id
    else:
        volume_id = find_volume_id(mountpoint=args.volume_id)
        args.unmount = True
    if args.unmount:
        cmd = "umount {devnode} || (kill -9 $(lsof -t +f -- $(readlink -f {devnode}) | sort | uniq); umount {devnode} || umount -l {devnode})"  # noqa
        subprocess.call(cmd.format(devnode=find_devnode(volume_id)), shell=True)
    attachment = resources.ec2.Volume(volume_id).attachments[0]
    res = clients.ec2.detach_volume(DryRun=args.dry_run,
                                    VolumeId=volume_id,
                                    InstanceId=attachment["InstanceId"],
                                    Device=attachment["Device"],
                                    Force=args.force)
    clients.ec2.get_waiter("volume_available").wait(VolumeIds=[volume_id])
    if args.delete:
        logger.info("Deleting EBS volume {}".format(volume_id))
        clients.ec2.delete_volume(VolumeId=volume_id, DryRun=args.dry_run)
    return res