Python os.mkdir() Examples

def create_model_directory(args):


    # create necessary directories
    if os.path.isdir("models_{name}".format(name=args.name)):
        print("Directory models_{name} already exists, old files will be overwritten".format(name=args.name), file=sys.stderr)
        
    else:
        os.mkdir("models_{name}".format(name=args.name))
        os.mkdir("models_{name}/Data".format(name=args.name))
        os.mkdir("models_{name}/Tokenizer".format(name=args.name))

    # copy necessary files
    if args.embeddings: # embeddings
        copyfile(args.embeddings, "models_{name}/Data/embeddings.vectors".format(name=args.name))
    copyfile("{config}/pipelines.yaml".format(config=args.config_directory), "models_{name}/pipelines.yaml".format(name=args.name))
    process_morpho(args) # train/dev files for tagger/parser
    process_config(args) # configs for tagger/parser 

def main(arguments):
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', help='directory to save data to', type=str, default='glue_data')
    parser.add_argument('--tasks', help='tasks to download data for as a comma separated string',
                        type=str, default='all')
    parser.add_argument('--path_to_mrpc',
                        help='path to directory containing extracted MRPC data, msr_paraphrase_train.txt and msr_paraphrase_text.txt',
                        type=str, default='')
    args = parser.parse_args(arguments)

    if not os.path.isdir(args.data_dir):
        os.mkdir(args.data_dir)
    tasks = get_tasks(args.tasks)

    for task in tasks:
        if task == 'MRPC':
            format_mrpc(args.data_dir, args.path_to_mrpc)
        elif task == 'diagnostic':
            download_diagnostic(args.data_dir)
        else:
            download_and_extract(task, args.data_dir) 

def download(url, filename, cache_directory):
    filename_cache = url.split('/')[-1]
    filename_cache = ''.join([c for c in filename_cache if c.isdigit() or c.isalpha()])
    filename_cache = cache_directory + "/" + filename_cache
    if os.path.exists(filename):
        return
    elif os.path.exists(filename_cache):
        print("Already downloaded")
        shutil.copyfile(filename_cache, filename)
    else:
        print("\nDownloading {} from {}".format(filename, url))
        os.mkdir(cache_directory)
        # wget.download(url, out=filename_cache)
        obj = SmartDL(url, filename_cache)
        obj.start()
        shutil.copyfile(filename_cache, filename) 

def plot_alignment(alignment, gs, dir=hp.logdir):
    """Plots the alignment.

    Args:
      alignment: A numpy array with shape of (encoder_steps, decoder_steps)
      gs: (int) global step.
      dir: Output path.
    """
    if not os.path.exists(dir): os.mkdir(dir)

    fig, ax = plt.subplots()
    im = ax.imshow(alignment)

    fig.colorbar(im)
    plt.title('{} Steps'.format(gs))
    plt.savefig('{}/alignment_{}.png'.format(dir, gs), format='png')
    plt.close(fig) 

def test_extraction():
    import shutil

    src = 'c:/Users/shinj/Downloads/clonezilla-live-2.5.2-31-amd64.iso'
    tmp_dir = 'c:/Users/shinj/Documents/tmp'
    for subdir, pattern in [
            ('single_string', 'EFI/'),
            ('single_list', ['EFI/']),
            ('multi', ['EFI/', 'syslinux/']),
            ('all', None) ]:
        dest_dir = os.path.join(tmp_dir, subdir)
        if os.path.exists(dest_dir):
            shutil.rmtree(dest_dir)
        os.mkdir(dest_dir)
        args = [src, dest_dir]
        if pattern is not None:
            args.append(pattern)
        print ('Calling extract_iso(%s)' % args)
        extract_iso(*args) 

def merge_bam_files(bams_per_genome, out, threads):
    """
    Merges (+sort +index)  all given bam files per genome (exact paths, single sample/multiple runs or multiple samples)
    """
    out_path = os.path.join(out,"bam")
    os.mkdir(out_path)
    for genome in bams_per_genome:
        list_of_bam = " ".join(bams_per_genome[genome]) # can be used as input to samtools immediately
        header = fix_headers(genome, bams_per_genome[genome], out_path)
        if header is not None:
            for bam in bams_per_genome[genome]: # add new header to all bam files
                cmd = "samtools reheader {header} {bam} >> {out}/out.bam; mv {out}/out.bam {bam}".format(
                    header = header,
                    out = out_path,
                    bam = bam
                )
                subprocess.call([cmd],shell=True)
        cmd = "samtools merge -@ {threads} - {bam_files} | samtools sort -@ {threads} - {path}/{genome}; samtools index {path}/{genome}.bam".format(
            threads = threads,
            bam_files = list_of_bam,
            path = out_path,
            genome = genome
        )
        subprocess.call([cmd],shell=True) # this runs a single command at a time (but that one multi threaded)
    return out_path 

def _prepare_simulation_subfolder(self, directory_strains):
		"""
		Create strain directory and copy templates and parameter file into it.

		@param directory_strains: Directory for the simulated strains
		@type directory_strains: str | unicode

		@return: Nothing
		@rtype: None
		"""
		if not os.path.exists(directory_strains):
			os.mkdir(directory_strains)
		for filename in self._directory_template_filenames:
			src = os.path.join(self._directory_template, filename)
			dst = os.path.join(directory_strains, filename)
			shutil.copy(src, dst) 

def __init__(self, config_file):
        # config = ConfigParser()
        super().__init__()

        self.test = None
        self.train = None
        config = myconf()
        config.read(config_file)
        self._config = config
        self.config_file = config_file

        print('Loaded config file sucessfully.')
        for section in config.sections():
            for k, v in config.items(section):
                print(k, ":", v)
        if not os.path.isdir(self.save_direction):
            os.mkdir(self.save_direction)
        config.write(open(config_file, 'w')) 

def get_payload_output(payload_output_dir):
	""" Builds directory structure if output option is supplied """
	output_dir = payload_output_dir
	# check to see if the trailing slash has been added to the path : ie /root/path
	if not output_dir.endswith("/"):
		output_dir = output_dir + "/"

	# creates the structure if it doesn't exist
	if not os.path.isdir(output_dir):
		print(yellowtxt("[!] Creating output directory structure"))
		os.mkdir(output_dir)
		os.chdir(output_dir)
		os.mkdir('handlers')
	return output_dir



###############################
### 	Helper Function	    ###
############################### 

def evaluate_detections(self, detections):
        """
        top level evaluations
        Parameters:
        ----------
        detections: list
            result list, each entry is a matrix of detections
        Returns:
        ----------
            None
        """
        # make all these folders for results
        result_dir = os.path.join(self.devkit_path, 'results')
        if not os.path.exists(result_dir):
            os.mkdir(result_dir)
        year_folder = os.path.join(self.devkit_path, 'results', 'VOC' + self.year)
        if not os.path.exists(year_folder):
            os.mkdir(year_folder)
        res_file_folder = os.path.join(self.devkit_path, 'results', 'VOC' + self.year, 'Main')
        if not os.path.exists(res_file_folder):
            os.mkdir(res_file_folder)

        self.write_pascal_results(detections)
        self.do_python_eval() 

def __init__(self, name, root_path):
        """
        basic information about an image database
        :param root_path: root path store cache and proposal data
        """
        self._name = name
        self._root_path = root_path

        # abstract attributes
        self._classes = []
        self._roidb = []

        # create cache
        cache_folder = os.path.join(self._root_path, 'cache')
        if not os.path.exists(cache_folder):
            os.mkdir(cache_folder) 

def copy_dependencies(f):
    config_path = '/etc/yangcatalog/yangcatalog.conf'
    config = ConfigParser.ConfigParser()
    config._interpolation = ConfigParser.ExtendedInterpolation()
    config.read(config_path)
    yang_models = config.get('Directory-Section', 'save-file-dir')
    tmp = config.get('Directory-Section', 'temp')
    out = f.getvalue()
    letters = string.ascii_letters
    suffix = ''.join(random.choice(letters) for i in range(8))
    dep_dir = '{}/yangvalidator-dependencies-{}'.format(tmp, suffix)
    os.mkdir(dep_dir)
    dependencies = out.split(':')[1].strip().split(' ')
    for dep in dependencies:
        for file in glob.glob(r'{}/{}*.yang'.format(yang_models, dep)):
            shutil.copy(file, dep_dir)
    return dep_dir 

def download(name, root_path=default_root_path):
    if not os.path.exists(os.path.join(root_path, 'benchmark')):
        os.mkdir(os.path.join(root_path, 'benchmark'))
    if not os.path.exists(os.path.join(root_path, 'pretrain')):
        os.mkdir(os.path.join(root_path, 'pretrain'))
    if name == 'nyt10':
        download_nyt10(root_path=root_path)
    elif name == 'semeval':
        download_semeval(root_path=root_path)
    elif name == 'wiki80':
        download_wiki80(root_path=root_path)
    elif name == 'glove':
        download_glove(root_path=root_path)
    elif name == 'bert_base_uncased':
        download_bert_base_uncased(root_path=root_path)
    else:
        raise Exception('Cannot find corresponding data.') 

def GetBaseImage(self, baseimage=None):
    """Downloads the base installer dmg."""
    baseos_path = os.path.join(self.cwd, BUILD,
                               'BaseOS')
    baseos_dmg = os.path.join(self.cwd, BUILD,
                              'BaseOS/Mac OS X Install DVD.dmg')
    try:
      os.mkdir(baseos_path)
    except OSError:
      pass
    if not os.path.exists(baseos_dmg):
      print 'Base image not found, getting latest one.'
      if not baseimage:
        src = os.path.join(self.webserver, 'osx_base',
                           '%s-default-base.dmg' % self.os_version)
      else:
        src = baseimage
      tgt = os.path.join(self.cwd, BUILD, 'BaseOS/Mac OS X Install DVD.dmg')

      self.DownloadFile(src, tgt) 

def GetBuildPackages(self):
    """Downloads the packages to be installed."""
    package_path = os.path.join(self.cwd, BUILD, 'Packages/')
    try:
      os.mkdir(package_path)
    except OSError:
      pass
    catalogs = [os.path.join(self.cwd, 'base%s_new.catalog' % self.os_version),
                os.path.join(self.cwd,
                             'thirdparty%s_new.catalog' % self.os_version)]

    for catalog in catalogs:
      f = open(catalog, 'r')
      packages = f.readlines()
      for line in packages:
        shutil.copy(os.path.join(TMPDIR, line.split()[0]),
                    os.path.join(package_path, line.split()[0])) 

def test_dyld_library_path_lookups():
    # Test that DYLD_LIBRARY_PATH can be used to find libs during
    # delocation
    with TempDirWithoutEnvVars('DYLD_LIBRARY_PATH') as tmpdir:
        # Copy libs into a temporary directory
        subtree = pjoin(tmpdir, 'subtree')
        all_local_libs = _make_libtree(subtree)
        liba, libb, libc, test_lib, slibc, stest_lib = all_local_libs
        # move libb and confirm that test_lib doesn't work
        hidden_dir = 'hidden'
        os.mkdir(hidden_dir)
        new_libb = os.path.join(hidden_dir, os.path.basename(LIBB))
        shutil.move(libb,
                    new_libb)
        assert_raises(RuntimeError, back_tick, [test_lib])
        # Update DYLD_LIBRARY_PATH and confirm that we can now
        # successfully delocate test_lib
        os.environ['DYLD_LIBRARY_PATH'] = hidden_dir
        delocate_path('subtree', 'deplibs')
        back_tick(test_lib) 

def test_dyld_library_path_beats_basename():
    # Test that we find libraries on DYLD_LIBRARY_PATH before basename
    with TempDirWithoutEnvVars('DYLD_LIBRARY_PATH') as tmpdir:
        # Copy libs into a temporary directory
        subtree = pjoin(tmpdir, 'subtree')
        all_local_libs = _make_libtree(subtree)
        liba, libb, libc, test_lib, slibc, stest_lib = all_local_libs
        # Copy liba into a subdirectory
        subdir = os.path.join(subtree, 'subdir')
        os.mkdir(subdir)
        new_libb = os.path.join(subdir, os.path.basename(LIBB))
        shutil.copyfile(libb, new_libb)
        # Without updating the environment variable, we find the lib normally
        predicted_lib_location = search_environment_for_lib(libb)
        # tmpdir can end up in /var, and that can be symlinked to
        # /private/var, so we'll use realpath to resolve the two
        assert_equal(predicted_lib_location, os.path.realpath(libb))
        # Updating shows us the new lib
        os.environ['DYLD_LIBRARY_PATH'] = subdir
        predicted_lib_location = search_environment_for_lib(libb)
        assert_equal(predicted_lib_location, new_libb) 

def test_dyld_fallback_library_path_loses_to_basename():
    # Test that we find libraries on basename before DYLD_FALLBACK_LIBRARY_PATH
    with TempDirWithoutEnvVars('DYLD_FALLBACK_LIBRARY_PATH') as tmpdir:
        # Copy libs into a temporary directory
        subtree = pjoin(tmpdir, 'subtree')
        all_local_libs = _make_libtree(subtree)
        liba, libb, libc, test_lib, slibc, stest_lib = all_local_libs
        # Copy liba into a subdirectory
        subdir = 'subdir'
        os.mkdir(subdir)
        new_libb = os.path.join(subdir, os.path.basename(LIBB))
        shutil.copyfile(libb, new_libb)
        os.environ['DYLD_FALLBACK_LIBRARY_PATH'] = subdir
        predicted_lib_location = search_environment_for_lib(libb)
        # tmpdir can end up in /var, and that can be symlinked to
        # /private/var, so we'll use realpath to resolve the two
        assert_equal(predicted_lib_location, os.path.realpath(libb)) 

def test_find_package_dirs():
    # Test utility for finding package directories
    with InTemporaryDirectory():
        os.mkdir('to_test')
        a_dir = pjoin('to_test', 'a_dir')
        b_dir = pjoin('to_test', 'b_dir')
        c_dir = pjoin('to_test', 'c_dir')
        for dir in (a_dir, b_dir, c_dir):
            os.mkdir(dir)
        assert_equal(find_package_dirs('to_test'), set([]))
        _write_file(pjoin(a_dir, '__init__.py'), "# a package")
        assert_equal(find_package_dirs('to_test'), {a_dir})
        _write_file(pjoin(c_dir, '__init__.py'), "# another package")
        assert_equal(find_package_dirs('to_test'), {a_dir, c_dir})
        # Not recursive
        assert_equal(find_package_dirs('.'), set())
        _write_file(pjoin('to_test', '__init__.py'), "# base package")
        # Also - strips '.' for current directory
        assert_equal(find_package_dirs('.'), {'to_test'}) 

def test_fuse_wheels():
    # Some tests for wheel fusing
    with InTemporaryDirectory():
        zip2dir(PLAT_WHEEL, 'to_wheel')
        zip2dir(PLAT_WHEEL, 'from_wheel')
        dir2zip('to_wheel', 'to_wheel.whl')
        dir2zip('from_wheel', 'from_wheel.whl')
        code, stdout, stderr = run_command(
            ['delocate-fuse', 'to_wheel.whl', 'from_wheel.whl'])
        assert_equal(code, 0)
        zip2dir('to_wheel.whl', 'to_wheel_fused')
        assert_same_tree('to_wheel_fused', 'from_wheel')
        # Test output argument
        os.mkdir('wheels')
        code, stdout, stderr = run_command(
            ['delocate-fuse', 'to_wheel.whl', 'from_wheel.whl',
             '-w', 'wheels'])
        zip2dir(pjoin('wheels', 'to_wheel.whl'), 'to_wheel_refused')
        assert_same_tree('to_wheel_refused', 'from_wheel') 

def save(filename):
    try:
        os.mkdir('saves/')
    except OSError:
        pass
    try:
        data = unicornhathd.get_pixels()
        data = repr(data)
        data = data.replace('array', 'list')
        print(filename, data)
        file = open('saves/' + filename + '.py', 'w')
        file.write("""#!/usr/bin/env python
import unicornhathd
import signal

unicornhathd.rotation(0)

pixels = {}

for x in range(unicornhathd.WIDTH):
    for y in range(unicornhathd.HEIGHT):
        r, g, b = pixels[x][y]
        unicornhathd.set_pixel(x, y, r, g, b)

unicornhathd.show()

print("\\nShowing: {}\\nPress Ctrl+C to exit!")

signal.pause()
""".format(data, filename))
        file.close()
        os.chmod('saves/' + filename + '.py', 0o777 | stat.S_IEXEC)

        return("ok" + str(unicornhathd.get_pixels()))
    except AttributeError:
        print("Unable to save, please update")
        print("unicornhathdhathd library!")
        return("fail") 

def download_diagnostic(data_dir):
    print("Downloading and extracting diagnostic...")
    if not os.path.isdir(os.path.join(data_dir, "diagnostic")):
        os.mkdir(os.path.join(data_dir, "diagnostic"))
    data_file = os.path.join(data_dir, "diagnostic", "diagnostic.tsv")
    urllib.request.urlretrieve(TASK2PATH["diagnostic"], data_file)
    print("\tCompleted!")
    return 

def _init_child(tmpdir):
    '''
        Initialize new process from multiprocessing module's Pool.

    :param tmpdir: Path of local staging area.
    '''
    signal.signal(signal.SIGINT, signal.SIG_IGN)

    # .................................for each process, create a isolated temp folder
    proc_dir = os.path.join(tmpdir, current_process().name)

    if not os.path.isdir(proc_dir):
        os.mkdir(proc_dir) 

def store_matrix(matrix='',
                 output_dir_path='',
                 out_file_name='',
                 output_format=''):
    """store_matrix."""
    if not os.path.exists(output_dir_path):
        os.mkdir(output_dir_path)
    full_out_file_name = os.path.join(output_dir_path, out_file_name)
    if output_format == "MatrixMarket":
        if len(matrix.shape) == 1:
            raise Exception(
                "'MatrixMarket' format supports only 2D dimensional array\
                and not vectors")
        else:
            io.mmwrite(full_out_file_name, matrix, precision=None)
    elif output_format == "numpy":
        np.save(full_out_file_name, matrix)
    elif output_format == "joblib":
        joblib.dump(matrix, full_out_file_name)
    elif output_format == "text":
        with open(full_out_file_name, "w") as f:
            if len(matrix.shape) == 1:
                for x in matrix:
                    f.write("%s\n" % (x))
            else:
                raise Exception(
                    "'text' format supports only mono dimensional array\
                    and not matrices")
    logger.info("Written file: %s" % full_out_file_name) 

def dump(obj, output_dir_path='', out_file_name=''):
    """dump."""
    if not os.path.exists(output_dir_path):
        os.mkdir(output_dir_path)
    full_out_file_name = os.path.join(output_dir_path, out_file_name) + ".pkl"
    joblib.dump(obj, full_out_file_name) 

def save_output(text=None, output_dir_path=None, out_file_name=None):
    """save_output."""
    if not os.path.exists(output_dir_path):
        os.mkdir(output_dir_path)
    full_out_file_name = os.path.join(output_dir_path, out_file_name)
    with open(full_out_file_name, 'w') as f:
        for line in text:
            f.write("%s\n" % str(line).strip())
    logger.info("Written file: %s (%d lines)" %
                (full_out_file_name, len(text))) 

def _remote_init(working_dir):
    global pickle
    import pickle
    import sys
    import shutil
    import os
    if not os.path.isdir(working_dir):
        os.mkdir(working_dir)
    sys.path.append(working_dir)
    shutil.move("_common.py", working_dir)
    shutil.move("_gdb.py", working_dir)
    shutil.move("cmds.gdb", working_dir)
    # setup CERT exploitable
    exp_lib_dir = os.path.join(working_dir, "exploitable", "lib")
    os.makedirs(exp_lib_dir)
    shutil.move("exploitable.py", os.path.join(working_dir, "exploitable"))
    shutil.move("__init__.py", exp_lib_dir)
    shutil.move("analyzers.py", exp_lib_dir)
    shutil.move("classifier.py", exp_lib_dir)
    shutil.move("elf.py", exp_lib_dir)
    shutil.move("gdb_wrapper.py", exp_lib_dir)
    shutil.move("rules.py", exp_lib_dir)
    shutil.move("tools.py", exp_lib_dir)
    shutil.move("versions.py", exp_lib_dir)
    os.chdir(working_dir)
    global _common
    global _gdb
    import _common
    import _gdb 

def local_run():
    opts, arg_error = parse_args()
    if opts.verbose:
        log.getLogger().setLevel(log.DEBUG)
    proj_cls = load_project(opts.project_name)
    if opts.reduce:
        for r in opts.reduce:
            if r not in reducers:
                arg_error("unknown reducer: \"%r\"" % r)

    tmp_wd = os.getcwd()
    if os.path.isdir(opts.template_or_directory):
        test_dir = os.path.abspath(opts.template_or_directory)
        tests = [os.path.join(test_dir, test) for test in os.listdir(opts.template_or_directory)]
        run_folder = "%s_%s_dir_replay" % (time.strftime("%Y%m%d-%H%M%S"), opts.project_name)
    else:
        tests = [opts.template_or_directory]
        run_folder = "%s_%s_local" % (time.strftime("%Y%m%d-%H%M%S"), opts.project_name)

    os.mkdir(run_folder)
    for template_fn in tests:
        template_fn = os.path.abspath(template_fn)
        os.chdir(run_folder)
        main(opts.project_name, proj_cls(template_fn), run_folder, template_fn,
             opts.iterations, opts.min_aggr, opts.max_aggr, opts.keep_mutations, opts.timeout,
             opts.pickle_result, opts.reduce, opts.reducen)
        os.chdir(tmp_wd) 

def _eval_discovery(self, output_dir):
    annopath = os.path.join(
        self._devkit_path,
        'VOC' + self._year,
        'Annotations',
        '{:s}.xml')
    imagesetfile = os.path.join(
        self._devkit_path,
        'VOC' + self._year,
        'ImageSets',
        'Main',
        self._image_set + '.txt')
    cachedir = os.path.join(self._devkit_path, 'annotations_dis_cache')
    corlocs = []
    if not os.path.isdir(output_dir):
        os.mkdir(output_dir)
    for i, cls in enumerate(self._classes):
        if cls == '__background__':
            continue
        filename = self._get_voc_results_file_template().format(cls)
        corloc = dis_eval(
            filename, annopath, imagesetfile, cls, cachedir, ovthresh=0.5)
        corlocs += [corloc]
        print('CorLoc for {} = {:.4f}'.format(cls, corloc))
        with open(os.path.join(output_dir, cls + '_corloc.pkl'), 'wb') as f:
            pickle.dump({'corloc': corloc}, f)
    print('Mean CorLoc = {:.4f}'.format(np.mean(corlocs)))
    print('~~~~~~~~')
    print('Results:')
    for corloc in corlocs:
        print('{:.3f}'.format(corloc))
    print('{:.3f}'.format(np.mean(corlocs)))
    print('~~~~~~~~') 

def make_directory_if_needed(directory_path):
  if not os.path.isdir(directory_path):
    os.mkdir(directory_path)