Python os.makedirs() Examples

The following are 30 code examples of os.makedirs(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module os , or try the search function .
Example #1
Source File: demo.py    From svviz with MIT License 32 votes vote down vote up
def downloadDemo(which):
    try:
        downloadDir = tempfile.mkdtemp()
        archivePath = "{}/svviz-data.zip".format(downloadDir)

        # logging.info("Downloading...")
        downloadWithProgress("http://svviz.github.io/svviz/assets/examples/{}.zip".format(which), archivePath)
        
        logging.info("Decompressing...")
        archive = zipfile.ZipFile(archivePath)
        archive.extractall("{}".format(downloadDir))

        if not os.path.exists("svviz-examples"):
            os.makedirs("svviz-examples/")

        shutil.move("{temp}/{which}".format(temp=downloadDir, which=which), "svviz-examples/")
    except Exception as e:
        print("error downloading and decompressing example data: {}".format(e))
        return False

    if not os.path.exists("svviz-examples"):
        print("error finding example data after download and decompression")
        return False
    return True 
Example #2
Source File: ssm.py    From aegea with Apache License 2.0 9 votes vote down vote up
def ensure_session_manager_plugin():
    session_manager_dir = os.path.join(config.user_config_dir, "bin")
    PATH = os.environ.get("PATH", "") + ":" + session_manager_dir
    if shutil.which("session-manager-plugin", path=PATH):
        subprocess.check_call(["session-manager-plugin"], env=dict(os.environ, PATH=PATH))
    else:
        os.makedirs(session_manager_dir, exist_ok=True)
        target_path = os.path.join(session_manager_dir, "session-manager-plugin")
        if platform.system() == "Darwin":
            download_session_manager_plugin_macos(target_path=target_path)
        elif platform.linux_distribution()[0] == "Ubuntu":
            download_session_manager_plugin_linux(target_path=target_path)
        else:
            download_session_manager_plugin_linux(target_path=target_path, pkg_format="rpm")
        os.chmod(target_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
        subprocess.check_call(["session-manager-plugin"], env=dict(os.environ, PATH=PATH))
    return shutil.which("session-manager-plugin", path=PATH) 
Example #3
Source File: web.py    From wechat-alfred-workflow with MIT License 8 votes vote down vote up
def save_to_path(self, filepath):
        """Save retrieved data to file at ``filepath``.

        .. versionadded: 1.9.6

        :param filepath: Path to save retrieved data.

        """
        filepath = os.path.abspath(filepath)
        dirname = os.path.dirname(filepath)
        if not os.path.exists(dirname):
            os.makedirs(dirname)

        self.stream = True

        with open(filepath, 'wb') as fileobj:
            for data in self.iter_content():
                fileobj.write(data) 
Example #4
Source File: utils.py    From incubator-spot with Apache License 2.0 8 votes vote down vote up
def create_oa_folders(cls, type, date):

        # create date and ingest summary folder structure if they don't' exist.
        root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
        data_type_folder = "{0}/data/{1}/{2}"
        if not os.path.isdir(data_type_folder.format(root_path, type, date)): os.makedirs(
            data_type_folder.format(root_path, type, date))
        if not os.path.isdir(data_type_folder.format(root_path, type, "ingest_summary")): os.makedirs(
            data_type_folder.format(root_path, type, "ingest_summary"))

        # create ipynb folders.
        ipynb_folder = "{0}/ipynb/{1}/{2}".format(root_path, type, date)
        if not os.path.isdir(ipynb_folder): os.makedirs(ipynb_folder)

        # retun path to folders.
        data_path = data_type_folder.format(root_path, type, date)
        ingest_path = data_type_folder.format(root_path, type, "ingest_summary")
        return data_path, ingest_path, ipynb_folder 
Example #5
Source File: utils.py    From pytorch_NER_BiLSTM_CNN_CRF with Apache License 2.0 7 votes vote down vote up
def save_model_all(model, save_dir, model_name, epoch):
    """
    :param model:  nn model
    :param save_dir: save model direction
    :param model_name:  model name
    :param epoch:  epoch
    :return:  None
    """
    if not os.path.isdir(save_dir):
        os.makedirs(save_dir)
    save_prefix = os.path.join(save_dir, model_name)
    save_path = '{}_epoch_{}.pt'.format(save_prefix, epoch)
    print("save all model to {}".format(save_path))
    output = open(save_path, mode="wb")
    torch.save(model.state_dict(), output)
    # torch.save(model.state_dict(), save_path)
    output.close() 
Example #6
Source File: _qemu.py    From ALF with Apache License 2.0 6 votes vote down vote up
def _remote_init(working_dir):
    global pickle
    import pickle
    import sys
    import shutil
    import os
    if not os.path.isdir(working_dir):
        os.mkdir(working_dir)
    sys.path.append(working_dir)
    shutil.move("_common.py", working_dir)
    shutil.move("_gdb.py", working_dir)
    shutil.move("cmds.gdb", working_dir)
    # setup CERT exploitable
    exp_lib_dir = os.path.join(working_dir, "exploitable", "lib")
    os.makedirs(exp_lib_dir)
    shutil.move("exploitable.py", os.path.join(working_dir, "exploitable"))
    shutil.move("__init__.py", exp_lib_dir)
    shutil.move("analyzers.py", exp_lib_dir)
    shutil.move("classifier.py", exp_lib_dir)
    shutil.move("elf.py", exp_lib_dir)
    shutil.move("gdb_wrapper.py", exp_lib_dir)
    shutil.move("rules.py", exp_lib_dir)
    shutil.move("tools.py", exp_lib_dir)
    shutil.move("versions.py", exp_lib_dir)
    os.chdir(working_dir)
    global _common
    global _gdb
    import _common
    import _gdb 
Example #7
Source File: os_utils.py    From godot-mono-builds with MIT License 6 votes vote down vote up
def mkdir_p(path):
    if not os.path.exists(path):
        print('creating directory: ' + path)
        os.makedirs(path)


# Remove files and/or directories recursively 
Example #8
Source File: __init__.py    From aegea with Apache License 2.0 6 votes vote down vote up
def initialize():
    global config, parser
    from .util.printing import BOLD, RED, ENDC
    config = AegeaConfig(__name__, use_yaml=True, save_on_exit=False)
    if not os.path.exists(config.config_files[2]):
        config_dir = os.path.dirname(os.path.abspath(config.config_files[2]))
        try:
            os.makedirs(config_dir)
        except OSError as e:
            if not (e.errno == errno.EEXIST and os.path.isdir(config_dir)):
                raise
        shutil.copy(os.path.join(os.path.dirname(__file__), "user_config.yml"), config.config_files[2])
        logger.info("Wrote new config file %s with default values", config.config_files[2])
        config = AegeaConfig(__name__, use_yaml=True, save_on_exit=False)

    parser = argparse.ArgumentParser(
        description="{}: {}".format(BOLD() + RED() + __name__.capitalize() + ENDC(), fill(__doc__.strip())),
        formatter_class=AegeaHelpFormatter
    )
    parser.add_argument("--version", action="version", version="%(prog)s {}\n{} {}\n{}".format(
        __version__,
        platform.python_implementation(),
        platform.python_version(),
        platform.platform()
    ))

    def help(args):
        parser.print_help()
    register_parser(help) 
Example #9
Source File: dataset.py    From spleeter with MIT License 6 votes vote down vote up
def cache(self, dataset, cache, wait):
        """ Cache the given dataset if cache is enabled. Eventually waits for
        cache to be available (useful if another process is already computing
        cache) if provided wait flag is True.

        :param dataset: Dataset to be cached if cache is required.
        :param cache: Path of cache directory to be used, None if no cache.
        :param wait: If caching is enabled, True is cache should be waited.
        :returns: Cached dataset if needed, original dataset otherwise.
        """
        if cache is not None:
            if wait:
                while not exists(f'{cache}.index'):
                    get_logger().info(
                        'Cache not available, wait %s',
                        self.WAIT_PERIOD)
                    time.sleep(self.WAIT_PERIOD)
            cache_path = os.path.split(cache)[0]
            os.makedirs(cache_path, exist_ok=True)
            return dataset.cache(cache)
        return dataset 
Example #10
Source File: utils.py    From pytorch_NER_BiLSTM_CNN_CRF with Apache License 2.0 6 votes vote down vote up
def save_best_model(model, save_dir, model_name, best_eval):
    """
    :param model:  nn model
    :param save_dir:  save model direction
    :param model_name:  model name
    :param best_eval:  eval best
    :return:  None
    """
    if best_eval.current_dev_score >= best_eval.best_dev_score:
        if not os.path.isdir(save_dir): os.makedirs(save_dir)
        model_name = "{}.pt".format(model_name)
        save_path = os.path.join(save_dir, model_name)
        print("save best model to {}".format(save_path))
        # if os.path.exists(save_path):  os.remove(save_path)
        output = open(save_path, mode="wb")
        torch.save(model.state_dict(), output)
        # torch.save(model.state_dict(), save_path)
        output.close()
        best_eval.early_current_patience = 0


# adjust lr 
Example #11
Source File: build.py    From Traffic_sign_detection_YOLO with MIT License 6 votes vote down vote up
def savepb(self):
		"""
		Create a standalone const graph def that 
		C++	can load and run.
		"""
		darknet_pb = self.to_darknet()
		flags_pb = self.FLAGS
		flags_pb.verbalise = False
		
		flags_pb.train = False
		# rebuild another tfnet. all const.
		tfnet_pb = TFNet(flags_pb, darknet_pb)		
		tfnet_pb.sess = tf.Session(graph = tfnet_pb.graph)
		# tfnet_pb.predict() # uncomment for unit testing
		name = 'built_graph/{}.pb'.format(self.meta['name'])
		os.makedirs(os.path.dirname(name), exist_ok=True)
		#Save dump of everything in meta
		with open('built_graph/{}.meta'.format(self.meta['name']), 'w') as fp:
			json.dump(self.meta, fp)
		self.say('Saving const graph def to {}'.format(name))
		graph_def = tfnet_pb.sess.graph_def
		tf.train.write_graph(graph_def,'./', name, False) 
Example #12
Source File: worker.py    From neural-fingerprinting with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def fetch_attacks_data(self):
    """Initializes data necessary to execute attacks.

    This method could be called multiple times, only first call does
    initialization, subsequent calls are noop.
    """
    if self.attacks_data_initialized:
      return
    # init data from datastore
    self.submissions.init_from_datastore()
    self.dataset_batches.init_from_datastore()
    self.adv_batches.init_from_datastore()
    # copy dataset locally
    if not os.path.exists(LOCAL_DATASET_DIR):
      os.makedirs(LOCAL_DATASET_DIR)
    eval_lib.download_dataset(self.storage_client, self.dataset_batches,
                              LOCAL_DATASET_DIR,
                              os.path.join(LOCAL_DATASET_COPY,
                                           self.dataset_name, 'images'))
    # download dataset metadata
    self.read_dataset_metadata()
    # mark as initialized
    self.attacks_data_initialized = True 
Example #13
Source File: isodump3.py    From multibootusb with GNU General Public License v2.0 6 votes vote down vote up
def writeDir_r(self, det_dir, dire, pp, r, all_type):
        #gen.log "writeDir_r:(%s)"%(det_dir)
        dirs = self.readDirItems(dire.locExtent, dire.lenData)
        for d in dirs:
            if not d.fIdentifier in [".", ".."]:
                if (pp != None) and (pp.search(d.fIdentifier) == None):
                    match = False
                else:
                    match = True
                #gen.log "mathing %s, %s, (%x)"%(match, d.fIdentifier, d.fFlag)
                p = det_dir + "/" + d.fIdentifier
                if d.fFlag & 0x02 == 0x02:
                    if not os.path.exists(p):
                        os.makedirs(p, 0o777)
                    if r:
                        if match:
                            self.writeDir_r(p, d, None, r, all_type) # Don't need to match subdirectory.
                        else:
                            self.writeDir_r(p, d, pp, r, all_type)
                elif match:
                    self.writeFile(d, p, all_type)
            # if not d.fIdentifier end #
        # for d in dirs end # 
Example #14
Source File: demo_letter_duvenaud.py    From nmp_qc with MIT License 6 votes vote down vote up
def plot_examples(data_loader, model, epoch, plotter, ind = [0, 10, 20]):

    # switch to evaluate mode
    model.eval()

    for i, (g, h, e, target) in enumerate(data_loader):
        if i in ind:
            subfolder_path = 'batch_' + str(i) + '_t_' + str(int(target[0][0])) + '/epoch_' + str(epoch) + '/'
            if not os.path.isdir(args.plotPath + subfolder_path):
                os.makedirs(args.plotPath + subfolder_path)

            num_nodes = torch.sum(torch.sum(torch.abs(h[0, :, :]), 1) > 0)
            am = g[0, 0:num_nodes, 0:num_nodes].numpy()
            pos = h[0, 0:num_nodes, :].numpy()

            plotter.plot_graph(am, position=pos, fig_name=subfolder_path+str(i) + '_input.png')

            # Prepare input data
            if args.cuda:
                g, h, e, target = g.cuda(), h.cuda(), e.cuda(), target.cuda()
            g, h, e, target = Variable(g), Variable(h), Variable(e), Variable(target)

            # Compute output
            model(g, h, e, lambda cls, id: plotter.plot_graph(am, position=pos, cls=cls,
                                                          fig_name=subfolder_path+ id)) 
Example #15
Source File: dump_model_files.py    From models with MIT License 6 votes vote down vote up
def get_models_overall(exp_name, rbp):
    print("RBP: " + rbp)

    out_h5 = "{rbp}/model_files/model.h5".format(rbp=rbp)
    os.makedirs(os.path.dirname(out_h5), exist_ok=True)

    trials = CMongoTrials(DB_NAME, exp_name + "_" + rbp, ip=HOST)

    # no trials yet - return None
    if trials.n_ok() == 0:
        trials = CMongoTrials(DB_NAME[:-2], exp_name + "_" + rbp, ip=HOST)
        if trials.n_ok() == 0:
            raise Exception("No trials")
    print("N trials: {0}".format(trials.n_ok()))

    # get best trial parameters
    tid = trials.best_trial_tid()
    model_path = trials.get_trial(tid)["result"]["path"]["model"]
    copyfile(model_path, out_h5) 
Example #16
Source File: utils.py    From nmp_qc with MIT License 5 votes vote down vote up
def save_checkpoint(state, is_best, directory):

    if not os.path.isdir(directory):
        os.makedirs(directory)
    checkpoint_file = os.path.join(directory, 'checkpoint.pth')
    best_model_file = os.path.join(directory, 'model_best.pth')
    torch.save(state, checkpoint_file)
    if is_best:
        shutil.copyfile(checkpoint_file, best_model_file) 
Example #17
Source File: LogMetric.py    From nmp_qc with MIT License 5 votes vote down vote up
def __init__(self, log_dir):
        if not os.path.isdir(log_dir):
            # if the directory does not exist we create the directory
            os.makedirs(log_dir)
        else:                      
            # clean previous logged data under the same directory name
            self._remove(log_dir)

        # configure the project
        configure(log_dir)

        self.global_step = 0 
Example #18
Source File: Plotter.py    From nmp_qc with MIT License 5 votes vote down vote up
def __init__(self, plot_dir = './'):
        self.plotdir = plot_dir

        if os.path.isdir(plot_dir):
            # clean previous logged data under the same directory name
            self._remove(plot_dir)

        os.makedirs(plot_dir) 
Example #19
Source File: validate_and_copy_submissions.py    From neural-fingerprinting with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def validate_and_copy_one_submission(self, submission_path):
    """Validates one submission and copies it to target directory.

    Args:
      submission_path: path in Google Cloud Storage of the submission file
    """
    if os.path.exists(self.download_dir):
      shutil.rmtree(self.download_dir)
    os.makedirs(self.download_dir)
    if os.path.exists(self.validate_dir):
      shutil.rmtree(self.validate_dir)
    os.makedirs(self.validate_dir)
    logging.info('\n' + ('#' * 80) + '\n# Processing submission: %s\n'
                 + '#' * 80, submission_path)
    local_path = self.copy_submission_locally(submission_path)
    metadata = self.base_validator.validate_submission(local_path)
    if not metadata:
      logging.error('Submission "%s" is INVALID', submission_path)
      self.stats.add_failure()
      return
    submission_type = metadata['type']
    container_name = metadata['container_gpu']
    logging.info('Submission "%s" is VALID', submission_path)
    self.list_of_containers.add(container_name)
    self.stats.add_success(submission_type)
    if self.do_copy:
      submission_id = '{0:04}'.format(self.cur_submission_idx)
      self.cur_submission_idx += 1
      self.copy_submission_to_destination(submission_path,
                                          TYPE_TO_DIR[submission_type],
                                          submission_id)
      self.id_to_path_mapping[submission_id] = submission_path 
Example #20
Source File: model.py    From models with MIT License 5 votes vote down vote up
def dump_models():
    """Dump column names
    """
    model_names = [os.path.basename(os.path.dirname(x))
                   for x in glob("../*/model.yaml")
                   if "template" not in x and "merged" not in x]
    os.makedirs("model_files", exist_ok=True)
    with open("models.txt", "w") as f:
        f.write("\n".join(model_names)) 
Example #21
Source File: dataloader.py    From models with MIT License 5 votes vote down vote up
def ensure_dirs(fname):
    """Ensure that the basepath of the given file path exists.

    Args:
      fname: (full) file path
    """
    required_path = "/".join(fname.split("/")[:-1])
    if not os.path.exists(required_path):
        os.makedirs(required_path) 
Example #22
Source File: trainer.py    From Deep_VoiceChanger with MIT License 5 votes vote down vote up
def preview_convert(iterator_a, iterator_b, g_a, g_b, device, gla, dst):
    @chainer.training.make_extension()
    def make_preview(trainer):
        with chainer.using_config('train', False):
            with chainer.no_backprop_mode():
                x_a = iterator_a.next()
                x_a = convert.concat_examples(x_a, device)
                x_a = chainer.Variable(x_a)

                x_b = iterator_b.next()
                x_b = convert.concat_examples(x_b, device)
                x_b = chainer.Variable(x_b)

                x_ab = g_a(x_a)
                x_ba = g_b(x_b)

                x_bab = g_a(x_ba)
                x_aba = g_b(x_ab)

                preview_dir = '{}/preview'.format(dst)
                if not os.path.exists(preview_dir):
                    os.makedirs(preview_dir)
                image_dir = '{}/image'.format(dst)
                if not os.path.exists(image_dir):
                    os.makedirs(image_dir)

                names = ['a', 'ab', 'aba', 'b', 'ba', 'bab']
                images = [x_a, x_ab, x_aba, x_b, x_ba, x_bab]
                for n, i in zip(names, images):
                    i = cp.asnumpy(i.data)[:,:,padding:-padding,:].reshape(1, -1, 128)
                    image.save(image_dir+'/{}{}.jpg'.format(trainer.updater.epoch,n), i)
                    w = np.concatenate([gla.inverse(_i) for _i in dataset.reverse(i)])
                    dataset.save(preview_dir+'/{}{}.wav'.format(trainer.updater.epoch,n), 16000, w)

    return make_preview 
Example #23
Source File: calc.py    From aospy with Apache License 2.0 5 votes vote down vote up
def _save_files(self, data, dtype_out_time):
        """Save the data to netcdf files in direc_out."""
        path = self.path_out[dtype_out_time]
        if not os.path.isdir(self.dir_out):
            os.makedirs(self.dir_out)
        if 'reg' in dtype_out_time:
            try:
                reg_data = xr.open_dataset(path)
            except (EOFError, RuntimeError, IOError):
                reg_data = xr.Dataset()
            reg_data.update(data)
            data_out = reg_data
        else:
            data_out = data
        if isinstance(data_out, xr.DataArray):
            data_out = xr.Dataset({self.name: data_out})
        data_out.to_netcdf(path, engine='netcdf4') 
Example #24
Source File: audio_transfer_learning.py    From sklearn-audio-transfer-learning with ISC License 5 votes vote down vote up
def extract_features_wrapper(paths, path2gt, model='vggish', save_as=False):
    """Wrapper function for extracting features (MusiCNN, VGGish or OpenL3) per batch.
       If a save_as string argument is passed, the features wiil be saved in 
       the specified file.
    """
    if model == 'vggish':
        feature_extractor = extract_vggish_features
    elif model == 'openl3' or model == 'musicnn':
        feature_extractor = extract_other_features
    else:
        raise NotImplementedError('Current implementation only supports MusiCNN, VGGish and OpenL3 features')

    batch_size = config['batch_size']
    first_batch = True
    for batch_id in tqdm(range(ceil(len(paths)/batch_size))):
        batch_paths = paths[(batch_id)*batch_size:(batch_id+1)*batch_size]
        [x, y, refs] = feature_extractor(batch_paths, path2gt, model)
        if first_batch:
            [X, Y, IDS] = [x, y, refs]
            first_batch = False
        else:
            X = np.concatenate((X, x), axis=0)
            Y = np.concatenate((Y, y), axis=0)
            IDS = np.concatenate((IDS, refs), axis=0)
    
    if save_as:  # save data to file
        # create a directory where to store the extracted training features
        audio_representations_folder = DATA_FOLDER + 'audio_representations/'
        if not os.path.exists(audio_representations_folder):
            os.makedirs(audio_representations_folder)
        np.savez(audio_representations_folder + save_as, X=X, Y=Y, IDS=IDS)
        print('Audio features stored: ', save_as)

    return [X, Y, IDS] 
Example #25
Source File: misc.py    From disentangling_conditional_gans with MIT License 5 votes vote down vote up
def create_result_subdir(result_dir, desc):

    # Select run ID and create subdir.
    while True:
        run_id = 0
        for fname in glob.glob(os.path.join(result_dir, '*')):
            try:
                fbase = os.path.basename(fname)
                ford = int(fbase[:fbase.find('-')])
                run_id = max(run_id, ford + 1)
            except ValueError:
                pass

        result_subdir = os.path.join(result_dir, '%03d-%s' % (run_id, desc))
        try:
            os.makedirs(result_subdir)
            break
        except OSError:
            if os.path.isdir(result_subdir):
                continue
            raise

    print("Saving results to", result_subdir)
    set_output_log_file(os.path.join(result_subdir, 'log.txt'))

    # Export config.
    try:
        with open(os.path.join(result_subdir, 'config.txt'), 'wt') as fout:
            for k, v in sorted(config.__dict__.items()):
                if not k.startswith('_'):
                    fout.write("%s = %s\n" % (k, str(v)))
    except:
        pass

    return result_subdir 
Example #26
Source File: dataset_tool.py    From disentangling_conditional_gans with MIT License 5 votes vote down vote up
def extract(tfrecord_dir, output_dir):
    print('Loading dataset "%s"' % tfrecord_dir)
    tfutil.init_tf({'gpu_options.allow_growth': True})
    dset = dataset.TFRecordDataset(tfrecord_dir, max_label_size=0, repeat=False, shuffle_mb=0)
    tfutil.init_uninited_vars()
    
    print('Extracting images to "%s"' % output_dir)
    if not os.path.isdir(output_dir):
        os.makedirs(output_dir)
    idx = 0
    while True:
        if idx % 10 == 0:
            print('%d\r' % idx, end='', flush=True)
        try:
            images, labels = dset.get_minibatch_np(1)
        except tf.errors.OutOfRangeError:
            break
        if images.shape[1] == 1:
            img = PIL.Image.fromarray(images[0][0], 'L')
        else:
            img = PIL.Image.fromarray(images[0].transpose(1, 2, 0), 'RGB')
        img.save(os.path.join(output_dir, 'img%08d.png' % idx))
        idx += 1
    print('Extracted %d images.' % idx)

#---------------------------------------------------------------------------- 
Example #27
Source File: dataset_tool.py    From disentangling_conditional_gans with MIT License 5 votes vote down vote up
def __init__(self, tfrecord_dir, expected_images, print_progress=True, progress_interval=10):
        self.tfrecord_dir       = tfrecord_dir
        self.tfr_prefix         = os.path.join(self.tfrecord_dir, os.path.basename(self.tfrecord_dir))
        self.expected_images    = expected_images
        self.cur_images         = 0
        self.shape              = None
        self.resolution_log2    = None
        self.tfr_writers        = []
        self.print_progress     = print_progress
        self.progress_interval  = progress_interval
        if self.print_progress:
            print('Creating dataset "%s"' % tfrecord_dir)
        if not os.path.isdir(self.tfrecord_dir):
            os.makedirs(self.tfrecord_dir)
        assert(os.path.isdir(self.tfrecord_dir)) 
Example #28
Source File: misc.py    From Random-Erasing with Apache License 2.0 5 votes vote down vote up
def mkdir_p(path):
    '''make dir if not exist'''
    try:
        os.makedirs(path)
    except OSError as exc:  # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise 
Example #29
Source File: news_corpus_generator.py    From news-corpus-builder with MIT License 5 votes vote down vote up
def _create_corpus_dir(self,directory):

        if not os.path.exists(directory):
            os.makedirs(directory) 
Example #30
Source File: start.py    From Starx_Pixiv_Collector with MIT License 5 votes vote down vote up
def download_thread(url, path, exfile_name=None, exfile_dir=None):
    tag = 'Download_Thread'
    wait_for_limit()
    local_path = path
    give_it_a_sign = False
    local_filename = url.split('/')[-1]
    if local_filename.endswith('zip'):
        give_it_a_sign = True
    if exfile_dir is not None:
        local_path += exfile_dir + global_symbol
    if exfile_name is not None:
        local_filename = exfile_name + "-" + local_filename
    path_output = local_path + local_filename
    print_with_tag(tag, ["File Location:" + path_output])
    if not os.path.exists(local_path):
        print_with_tag(tag, "Folder doesn't exists!!")
        os.makedirs(local_path)
        print_with_tag(tag, "Folder Created.")

    retry_count = 0
    while True:
        try:
            _thread.TIMEOUT_MAX = 60
            _thread.start_new_thread(download_file, (url, path_output, give_it_a_sign))
        except:
            print_with_tag(tag, "Error.")
            if retry_count == 3:
                print_with_tag(tag, "Not wokring..")
                print_with_tag(tag, "Skip!!")
            else:
                print_with_tag(tag, "Starting retry..")
                retry_count += 1
        else:
            print_with_tag(tag, "Download thread successfully started!")
            break
    print_with_tag(tag, ['Threads_count:', str(current_threads)])