Python os.makedirs() Examples

The following are 50 code examples for showing how to use os.makedirs(). They are extracted from open source Python projects. You can vote up the examples you like or vote down the exmaples you don't like. You can also save this page to your account.

Example 1
Project: charm-plumgrid-gateway   Author: openstack   File: charm_helpers_sync.py View Source Project 8 votes vote down vote up
def sync_helpers(include, src, dest, options=None):
    if not os.path.isdir(dest):
        os.makedirs(dest)

    global_options = parse_sync_options(options)

    for inc in include:
        if isinstance(inc, str):
            inc, opts = extract_options(inc, global_options)
            sync(src, dest, inc, opts)
        elif isinstance(inc, dict):
            # could also do nested dicts here.
            for k, v in six.iteritems(inc):
                if isinstance(v, list):
                    for m in v:
                        inc, opts = extract_options(m, global_options)
                        sync(src, dest, '%s.%s' % (k, inc), opts) 
Example 2
Project: nanostat   Author: wdecoster   File: NanoStat.py View Source Project 8 votes vote down vote up
def main():
    args = get_args()
    if not os.path.exists(args.outdir):
        os.makedirs(args.outdir)
    sources = [args.fastq, args.bam, args.summary]
    sourcename = ["fastq", "bam", "summary"]
    datadf = nanoget.get_input(
        source=[n for n, s in zip(sourcename, sources) if s][0],
        files=[f for f in sources if f][0],
        threads=args.threads,
        readtype=args.readtype,
        combine="track")
    if args.name:
        output = args.name
    else:
        output = os.path.join(args.outdir, args.prefix + "NanoStats.txt")
    write_stats([datadf], output) 
Example 3
Project: charm-plumgrid-gateway   Author: openstack   File: host.py View Source Project 7 votes vote down vote up
def mkdir(path, owner='root', group='root', perms=0o555, force=False):
    """Create a directory"""
    log("Making dir {} {}:{} {:o}".format(path, owner, group,
                                          perms))
    uid = pwd.getpwnam(owner).pw_uid
    gid = grp.getgrnam(group).gr_gid
    realpath = os.path.abspath(path)
    path_exists = os.path.exists(realpath)
    if path_exists and force:
        if not os.path.isdir(realpath):
            log("Removing non-directory file {} prior to mkdir()".format(path))
            os.unlink(realpath)
            os.makedirs(realpath, perms)
    elif not path_exists:
        os.makedirs(realpath, perms)
    os.chown(realpath, uid, gid)
    os.chmod(realpath, perms) 
Example 4
Project: safetyculture-sdk-python   Author: SafetyCulture   File: exporter.py View Source Project 7 votes vote down vote up
def create_directory_if_not_exists(logger, path):
    """
    Creates 'path' if it does not exist

    If creation fails, an exception will be thrown

    :param logger:  the logger
    :param path:    the path to ensure it exists
    """
    try:
        os.makedirs(path)
    except OSError as ex:
        if ex.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            log_critical_error(logger, ex, 'An error happened trying to create ' + path)
            raise 
Example 5
Project: safetyculture-sdk-python   Author: SafetyCulture   File: exporter.py View Source Project 7 votes vote down vote up
def save_exported_media_to_file(logger, export_dir, media_file, filename, extension):
    """
    Write exported media item to disk at specified location with specified file name.
    Any existing file with the same name will be overwritten.
    :param logger:      the logger
    :param export_dir:  path to directory for exports
    :param media_file:  media file to write to disc
    :param filename:    filename to give exported image
    :param extension:   extension to give exported image
    """
    if not os.path.exists(export_dir):
        logger.info("Creating directory at {0} for media files.".format(export_dir))
        os.makedirs(export_dir)
    file_path = os.path.join(export_dir, filename + '.' + extension)
    if os.path.isfile(file_path):
        logger.info('Overwriting existing report at ' + file_path)
    try:
        with open(file_path, 'wb') as out_file:
            shutil.copyfileobj(media_file.raw, out_file)
        del media_file
    except Exception as ex:
        log_critical_error(logger, ex, 'Exception while writing' + file_path + ' to file') 
Example 6
Project: alfred-mpd   Author: deanishe   File: web.py View Source Project 6 votes vote down vote up
def save_to_path(self, filepath):
        """Save retrieved data to file at ``filepath``.

        .. versionadded: 1.9.6

        :param filepath: Path to save retrieved data.

        """
        filepath = os.path.abspath(filepath)
        dirname = os.path.dirname(filepath)
        if not os.path.exists(dirname):
            os.makedirs(dirname)

        self.stream = True

        with open(filepath, 'wb') as fileobj:
            for data in self.iter_content():
                fileobj.write(data) 
Example 7
Project: safetyculture-sdk-python   Author: SafetyCulture   File: import_grs.py View Source Project 6 votes vote down vote up
def create_directory_if_not_exists(logger, path):
    """
    Creates 'path' if it does not exist

    If creation fails, an exception will be thrown

    :param logger:  the logger
    :param path:    the path to ensure it exists
    """
    try:
        os.makedirs(path)
    except OSError as ex:
        if ex.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            log_critical_error(logger, ex, 'An error happened trying to create ' + path)
            raise 
Example 8
Project: safetyculture-sdk-python   Author: SafetyCulture   File: safetypy.py View Source Project 6 votes vote down vote up
def create_directory_if_not_exists(self, path):
        """
        Creates 'path' if it does not exist

        If creation fails, an exception will be thrown

        :param path:    the path to ensure it exists
        """
        try:
            os.makedirs(path)
        except OSError as ex:
            if ex.errno == errno.EEXIST and os.path.isdir(path):
                pass
            else:
                self.log_critical_error(ex, 'An error happened trying to create ' + path)
                raise 
Example 9
Project: Causality   Author: vcla   File: dealWithDBResults.py View Source Project 6 votes vote down vote up
def downloadExamples(examples,connType,conn=False):
	print("===========")
	print("DOWNLOADING")
	print("===========")
	try:
		os.makedirs(kResultStorageFolder)
	except OSError:
		if not os.path.isdir(kResultStorageFolder):
			raise
	leaveconn = True
	if not conn:
		leaveconn = False
		conn = getDB(connType)
	for example in examples:
		print("---------\nEXAMPLE: {}\n-------".format(example))
		example, room = example.rsplit('_',1)
		getExampleFromDB(example, connType, conn)
	if not leaveconn:
		conn.close() 
Example 10
Project: python-   Author: secondtonone1   File: output.py View Source Project 6 votes vote down vote up
def workthread(item, user_agent,path):
	strurl = 'http://yxpjw.club'+item[0]
	picname = item[1]
	print('????%s...........................\n' %(picname))
	req = request.Request(strurl)
	req.add_header('User-Agent',user_agent)
	response = request.urlopen(req)
	content = response.read().decode('gbk')
	strurl2 = re.search(r'^(.*)/',strurl).group(0)
	print('https headers...............%s'%(strurl2))
	#destname = os.path.join(path,picname+'.txt')
	#with open(destname, 'w',encoding='gbk') as file:
		#file.write(content)
	destdir = os.path.join(path,picname)
	os.makedirs(destdir)
	page = 1
	while(1):
		content = getpagedata(content,destdir,page,strurl2)
		if not content:
			break
		page = page + 1
	print('%s?????????\n'%(picname)) 
Example 11
Project: python-   Author: secondtonone1   File: resources.py View Source Project 6 votes vote down vote up
def get(self, resource):
        """
        Get a resource into the cache,

        :param resource: A :class:`Resource` instance.
        :return: The pathname of the resource in the cache.
        """
        prefix, path = resource.finder.get_cache_info(resource)
        if prefix is None:
            result = path
        else:
            result = os.path.join(self.base, self.prefix_to_dir(prefix), path)
            dirname = os.path.dirname(result)
            if not os.path.isdir(dirname):
                os.makedirs(dirname)
            if not os.path.exists(result):
                stale = True
            else:
                stale = self.is_stale(resource, path)
            if stale:
                # write the bytes of the resource to the cache location
                with open(result, 'wb') as f:
                    f.write(resource.bytes)
        return result 
Example 12
Project: PyWallet   Author: AndreMiras   File: pywalib.py View Source Project 6 votes vote down vote up
def delete_account(self, account):
        """
        Deletes the given `account` from the `keystore_dir` directory.
        Then deletes it from the `AccountsService` account manager instance.
        In fact, moves it to another location; another directory at the same
        level.
        """
        app = self.app
        keystore_dir = app.services.accounts.keystore_dir
        deleted_keystore_dir = PyWalib.deleted_account_dir(keystore_dir)
        # create the deleted account dir if required
        if not os.path.exists(deleted_keystore_dir):
            os.makedirs(deleted_keystore_dir)
        # "removes" it from the file system
        account_filename = os.path.basename(account.path)
        deleted_account_path = os.path.join(
            deleted_keystore_dir, account_filename)
        shutil.move(account.path, deleted_account_path)
        # deletes it from the `AccountsService` account manager instance
        account_service = self.get_account_list()
        account_service.accounts.remove(account) 
Example 13
Project: PyWallet   Author: AndreMiras   File: test_pywalib.py View Source Project 6 votes vote down vote up
def test_delete_account_already_exists(self):
        """
        If the destination (backup/trash) directory where the account is moved
        already exists, it should be handled gracefully.
        This could happens if the account gets deleted, then reimported and
        deleted again, refs:
        https://github.com/AndreMiras/PyWallet/issues/88
        """
        pywalib = self.pywalib
        account = self.helper_new_account()
        # creates a file in the backup/trash folder that would conflict
        # with the deleted account
        deleted_keystore_dir = PyWalib.deleted_account_dir(self.keystore_dir)
        os.makedirs(deleted_keystore_dir)
        account_filename = os.path.basename(account.path)
        deleted_account_path = os.path.join(
            deleted_keystore_dir, account_filename)
        # create that file
        open(deleted_account_path, 'a').close()
        # then deletes the account and verifies it worked
        self.assertEqual(len(pywalib.get_account_list()), 1)
        pywalib.delete_account(account)
        self.assertEqual(len(pywalib.get_account_list()), 0) 
Example 14
Project: treecat   Author: posterior   File: generate.py View Source Project 6 votes vote down vote up
def generate_dataset_file(num_rows, num_cols, num_cats=4, rate=1.0):
    """Generate a random dataset.

    Returns:
        The path to a gzipped pickled data table.
    """
    path = os.path.join(DATA, '{}-{}-{}-{:0.1f}.dataset.pkz'.format(
        num_rows, num_cols, num_cats, rate))
    if os.path.exists(path):
        return path
    print('Generating {}'.format(path))
    if not os.path.exists(DATA):
        os.makedirs(DATA)
    dataset = generate_dataset(num_rows, num_cols, num_cats, rate)
    pickle_dump(dataset, path)
    return path 
Example 15
Project: treecat   Author: posterior   File: generate.py View Source Project 6 votes vote down vote up
def generate_model_file(num_rows, num_cols, num_cats=4, rate=1.0):
    """Generate a random model.

    Returns:
        The path to a gzipped pickled model.
    """
    path = os.path.join(DATA, '{}-{}-{}-{:0.1f}.model.pkz'.format(
        num_rows, num_cols, num_cats, rate))
    V = num_cols
    K = V * (V - 1) // 2
    if os.path.exists(path):
        return path
    print('Generating {}'.format(path))
    if not os.path.exists(DATA):
        os.makedirs(DATA)
    dataset_path = generate_dataset_file(num_rows, num_cols, num_cats, rate)
    dataset = pickle_load(dataset_path)
    table = dataset['table']
    tree_prior = np.zeros(K, dtype=np.float32)
    config = make_config(learning_init_epochs=5)
    model = train_model(table, tree_prior, config)
    pickle_dump(model, path)
    return path 
Example 16
Project: stalker_pyramid   Author: eoyilmaz   File: archive.py View Source Project 6 votes vote down vote up
def create_default_project(cls, path, name='DefaultProject'):
        """Creates default maya project structure along with a suitable
        workspace.mel file.

        :param str path: The path that the default project structure will be
          created.

        :return:
        """
        project_path = os.path.join(path, name)

        # lets create the structure
        for dir_name in cls.default_project_structure.split('\n'):
            dir_path = os.path.join(project_path, dir_name)
            try:
                os.makedirs(dir_path)
            except OSError:
                pass

        # create the workspace.mel
        workspace_mel_path = os.path.join(project_path, 'workspace.mel')
        with open(workspace_mel_path, 'w+') as f:
            f.writelines(cls.default_workspace_content)

        return project_path 
Example 17
Project: my-first-blog   Author: AnkurBegining   File: resources.py View Source Project 6 votes vote down vote up
def get(self, resource):
        """
        Get a resource into the cache,

        :param resource: A :class:`Resource` instance.
        :return: The pathname of the resource in the cache.
        """
        prefix, path = resource.finder.get_cache_info(resource)
        if prefix is None:
            result = path
        else:
            result = os.path.join(self.base, self.prefix_to_dir(prefix), path)
            dirname = os.path.dirname(result)
            if not os.path.isdir(dirname):
                os.makedirs(dirname)
            if not os.path.exists(result):
                stale = True
            else:
                stale = self.is_stale(resource, path)
            if stale:
                # write the bytes of the resource to the cache location
                with open(result, 'wb') as f:
                    f.write(resource.bytes)
        return result 
Example 18
Project: picoCTF   Author: picoCTF   File: deploy.py View Source Project 6 votes vote down vote up
def generate_instance_deployment_directory(username):
    """
    Generates the instance deployment directory for the given username
    """

    directory = username
    if deploy_config.obfuscate_problem_directories:
        directory = md5((username + deploy_config.deploy_secret).encode()).hexdigest()

    root_dir = deploy_config.problem_directory_root

    if not isdir(root_dir):
        os.makedirs(root_dir)
        # make the root not world readable
        os.chmod(root_dir, 0o751)

    path = join(root_dir, directory)
    if not isdir(path):
        os.makedirs(path)

    return path 
Example 19
Project: polyaxon-cli   Author: polyaxon   File: base.py View Source Project 6 votes vote down vote up
def get_config_file_path(cls):
        if not cls.IS_GLOBAL:
            # local to this directory
            base_path = os.path.join('.')
        else:
            base_path = os.path.expanduser('~')
            if not os.access(base_path, os.W_OK):
                base_path = '/tmp'

            base_path = os.path.join(base_path, '.polyaxon')

            if not os.path.exists(base_path):
                try:
                    os.makedirs(base_path)
                except OSError:
                    # Except permission denied and potential race conditions
                    # in multi-threaded environments.
                    logger.error('Could not create config directory `{}`'.format(base_path))

        return os.path.join(base_path, cls.CONFIG_FILE_NAME) 
Example 20
Project: sstash   Author: realcr   File: sstash.py View Source Project 6 votes vote down vote up
def read_dir(self,key,dest_dir):
        """
        Read a directory from a key (recursively)
        """
        if os.path.exists(dest_dir):
            raise SSError('Path {} already exists'.format(dest_dir))

        def inner_read_dir(prefix):
            os.makedirs(join(dest_dir,*prefix))
            for child in self.get_children(key + prefix):
                fullkey = key + prefix + [child]
                has_value = True
                try:
                    self.read_value(fullkey)
                except SSKeyError:
                    has_value = False

                if has_value:
                    self.read_file(fullkey,\
                            join(dest_dir,*(prefix + [child])))
                else:
                    inner_read_dir(prefix + [child])
        inner_read_dir([]) 
Example 21
Project: OldMunkiPackages   Author: aysiu   File: OldMunkiPackages.py View Source Project 6 votes vote down vote up
def trash_old_stuff(trashlist, trashpath, newpath):
	if isinstance(trashlist, list):
		for old_location in trashlist:
			# Get the subfolders needed to be created
			path_within_destination=os.path.relpath(old_location, trashpath)
			# Create what will be the destination path
			new_location=os.path.join(newpath, path_within_destination)
			# Make sure all the relevant subfolders exist in the destination
			if not os.path.exists(os.path.dirname(new_location)):
				os.makedirs(os.path.dirname(new_location))
			# Even though we've been double-checking paths all along, let's just make one last check
			if os.path.exists(old_location) and os.path.isdir(newpath):
				os.rename(old_location, new_location)
				logging.info("Moving %s to %s\n" % (old_location, new_location))
			else:
				logging.error("One of %s or %s does not exist\n" % (old_location, new_location))
	else:
		logging.error("%s is not a valid list\n" % trashlist)

# Function that checks paths are writable 
Example 22
Project: charm-swift-proxy   Author: openstack   File: host.py View Source Project 6 votes vote down vote up
def mkdir(path, owner='root', group='root', perms=0o555, force=False):
    """Create a directory"""
    log("Making dir {} {}:{} {:o}".format(path, owner, group,
                                          perms))
    uid = pwd.getpwnam(owner).pw_uid
    gid = grp.getgrnam(group).gr_gid
    realpath = os.path.abspath(path)
    path_exists = os.path.exists(realpath)
    if path_exists and force:
        if not os.path.isdir(realpath):
            log("Removing non-directory file {} prior to mkdir()".format(path))
            os.unlink(realpath)
            os.makedirs(realpath, perms)
    elif not path_exists:
        os.makedirs(realpath, perms)
    os.chown(realpath, uid, gid)
    os.chmod(realpath, perms) 
Example 23
Project: charm-swift-proxy   Author: openstack   File: host.py View Source Project 6 votes vote down vote up
def mkdir(path, owner='root', group='root', perms=0o555, force=False):
    """Create a directory"""
    log("Making dir {} {}:{} {:o}".format(path, owner, group,
                                          perms))
    uid = pwd.getpwnam(owner).pw_uid
    gid = grp.getgrnam(group).gr_gid
    realpath = os.path.abspath(path)
    path_exists = os.path.exists(realpath)
    if path_exists and force:
        if not os.path.isdir(realpath):
            log("Removing non-directory file {} prior to mkdir()".format(path))
            os.unlink(realpath)
            os.makedirs(realpath, perms)
    elif not path_exists:
        os.makedirs(realpath, perms)
    os.chown(realpath, uid, gid)
    os.chmod(realpath, perms) 
Example 24
Project: charm-swift-proxy   Author: openstack   File: nrpe.py View Source Project 6 votes vote down vote up
def copy_nrpe_checks():
    """
    Copy the nrpe checks into place

    """
    NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins'
    nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks',
                                  'charmhelpers', 'contrib', 'openstack',
                                  'files')

    if not os.path.exists(NAGIOS_PLUGINS):
        os.makedirs(NAGIOS_PLUGINS)
    for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")):
        if os.path.isfile(fname):
            shutil.copy2(fname,
                         os.path.join(NAGIOS_PLUGINS, os.path.basename(fname))) 
Example 25
Project: BioDownloader   Author: biomadeira   File: fetchers.py View Source Project 6 votes vote down vote up
def download_sifts_from_ebi(identifier, override=False):
    """
    Downloads a SIFTS xml from the EBI FTP to the filesystem.

    :param identifier: (str) PDB ID
    :param override: (boolean)
    :return: (side effects)
    """

    filename = "{}.xml.gz".format(identifier)
    outputfile = os.path.join(config.db_root, config.db_sifts, filename)
    os.makedirs(os.path.join(config.db_root, config.db_sifts), exist_ok=True)

    url_root = config.ftp_sifts
    url_endpoint = "{}.xml.gz".format(identifier)
    url = url_root + url_endpoint
    Downloader(url=url, outputfile=outputfile,
               decompress=True, override=override)
    return 
Example 26
Project: BioDownloader   Author: biomadeira   File: fetchers.py View Source Project 6 votes vote down vote up
def download_data_from_uniprot(identifier, file_format="fasta", override=False):
    """
    Downloads a UniProt fasta, gff or txt to the filesystem.

    :param identifier: (str) UniProt ID
    :param file_format: (str) endpoint
    :param override: (boolean)
    :return: (side effects)
    """

    file_format = file_format.lstrip('.')
    if file_format in ['txt', 'fasta', 'gff']:
        filename = "{}.{}".format(identifier, file_format)
        outputfile = os.path.join(config.db_root, config.db_uniprot, filename)
        os.makedirs(os.path.join(config.db_root, config.db_uniprot), exist_ok=True)

        url_root = config.http_uniprot
        url_endpoint = "{}.{}".format(identifier, file_format)
        url = url_root + url_endpoint
        Downloader(url=url, outputfile=outputfile,
                   decompress=True, override=override)
    else:
        raise ValueError("File format {} is not currently implemented..."
                         "".format(file_format))
    return 
Example 27
Project: BioDownloader   Author: biomadeira   File: fetchers.py View Source Project 6 votes vote down vote up
def download_alignment_from_pfam(identifier, alignment_size="seed",
                                 override=False):
    """
    Downloads a MSA in Stockholm format from Pfam to the filesystem.

    :param identifier: (str) PFam ID
    :param alignment_size: (str) either "seed" or "full"
    :param override: (boolean)
    :return: (side effects)
    """

    filename = "{}.sth".format(identifier)
    outputfile = os.path.join(config.db_root, config.db_pfam, filename)
    os.makedirs(os.path.join(config.db_root, config.db_pfam), exist_ok=True)

    url_root = config.http_pfam
    url_endpoint = ("family/{}/alignment/{}"
                    "".format(identifier, alignment_size))
    url = url_root + url_endpoint
    Downloader(url=url, outputfile=outputfile,
               decompress=True, override=override)
    return 
Example 28
Project: human-rl   Author: gsastry   File: classifier_tf.py View Source Project 6 votes vote down vote up
def predict_episodes(self, model, episode_paths, n=None, out_dir=None, prefix="model/"):
        if n is not None:
            episode_paths = np.random.choice(episode_paths, n, replace=False)
        if out_dir is not None:
            os.makedirs(out_dir, exist_ok=True)
        for ep, episode_path in enumerate(episode_paths):
            episode = frame.load_episode(episode_path)
            features = self.load_features_episode(episode)
            prediction = model.predict_proba(features)
            for i in range(len(prediction)):
                episode.frames[i].info[prefix + "score"] = prediction[i]
                episode.frames[i].info[prefix + "label"] = model.apply_threshold(prediction[i])
            out_path = episode_path
            if out_dir is not None:
                out_path = os.path.join(out_dir, "{}.pkl.gz".format(ep))
            frame.save_episode(out_path, episode) 
Example 29
Project: human-rl   Author: gsastry   File: classifier_tf.py View Source Project 6 votes vote down vote up
def predict_episodes(self, model, episode_paths, n=None, out_dir=None, prefix="model/"):
        if n is not None:
            episode_paths = np.random.choice(episode_paths, n, replace=False)
        if out_dir is not None:
            os.makedirs(out_dir, exist_ok=True)
        for ep, episode_path in enumerate(episode_paths):
            episode = frame.load_episode(episode_path)
            features = self.load_features_episode(episode)
            prediction = model.predict_proba(features)
            for i in range(len(prediction)):
                episode.frames[i].info[prefix + "score"] = prediction[i]
                episode.frames[i].info[prefix + "label"] = model.apply_threshold(prediction[i])
            out_path = episode_path
            if out_dir is not None:
                out_path = os.path.join(out_dir, "{}.pkl.gz".format(ep))
            frame.save_episode(out_path, episode) 
Example 30
Project: human-rl   Author: gsastry   File: classifier_tf.py View Source Project 6 votes vote down vote up
def predict_episodes(self, model, episode_paths, n=None, out_dir=None, prefix="model/"):
        if n is not None:
            episode_paths = np.random.choice(episode_paths, n, replace=False)
        if out_dir is not None:
            os.makedirs(out_dir, exist_ok=True)
        for ep, episode_path in enumerate(episode_paths):
            episode = frame.load_episode(episode_path)
            features = self.load_features_episode(episode)
            prediction = model.predict_proba(features)
            for i in range(len(prediction)):
                episode.frames[i].info[prefix + "score"] = prediction[i]
                episode.frames[i].info[prefix + "label"] = model.apply_threshold(prediction[i])
            out_path = episode_path
            if out_dir is not None:
                out_path = os.path.join(out_dir, "{}.pkl.gz".format(ep))
            frame.save_episode(out_path, episode) 
Example 31
Project: human-rl   Author: gsastry   File: build_training_data_tfrecord.py View Source Project 6 votes vote down vote up
def convert_episode_to_tf_records(base_directory, new_directory, dataloader, path):
    episode = frame.load_episode(path)
    features, labels = dataloader.load_features_and_labels_episode(episode)
    assert path.rfind(base_directory) > -1
    new_path = path[path.rfind(base_directory) + len(base_directory) + 1:]
    new_path = os.path.splitext(new_path)[0]
    new_path = os.path.splitext(new_path)[0]
    new_path = os.path.join(new_directory, new_path + ".tfrecord")
    options = tf.python_io.TFRecordOptions(
        compression_type=tf.python_io.TFRecordCompressionType.GZIP)
    os.makedirs(new_path, exist_ok=True)
    for i, f in enumerate(episode.frames):
        writer = tf.python_io.TFRecordWriter(
            os.path.join(new_path, "{}.tfrecord".format(i)), options=options)
        example = tf.train.Example(features=tf.train.Features(feature={
            'action': _int64_feature([f.action]),
            'label': _int64_feature([f.label] if f.label is not None else []),
            'observation': _float_feature(f.observation.reshape(-1)),
            'observation_shape': _int64_feature(f.observation.shape),
            'image': _bytes_feature([f.image.tobytes()]),
            'image_shape': _int64_feature(f.image.shape),
        }))
        writer.write(example.SerializeToString())
        writer.close()
    return new_path 
Example 32
Project: human-rl   Author: gsastry   File: classifier_tf.py View Source Project 6 votes vote down vote up
def predict_episodes(self, model, episode_paths, n=None, out_dir=None, prefix="model/"):
        if n is not None:
            episode_paths = np.random.choice(episode_paths, n, replace=False)
        if out_dir is not None:
            os.makedirs(out_dir, exist_ok=True)
        for ep, episode_path in enumerate(episode_paths):
            episode = frame.load_episode(episode_path)
            features = self.load_features_episode(episode)
            prediction = model.predict_proba(features)
            for i in range(len(prediction)):
                episode.frames[i].info[prefix + "score"] = prediction[i]
                episode.frames[i].info[prefix + "label"] = model.apply_threshold(prediction[i])
            out_path = episode_path
            if out_dir is not None:
                out_path = os.path.join(out_dir, "{}.pkl.gz".format(ep))
            frame.save_episode(out_path, episode) 
Example 33
Project: distributional_perspective_on_RL   Author: Kiwoo   File: logger.py View Source Project 6 votes vote down vote up
def make_output_format(format, ev_dir):
    os.makedirs(ev_dir, exist_ok=True)
    if format == 'stdout':
        return HumanOutputFormat(sys.stdout)
    elif format == 'log':
        log_file = open(osp.join(ev_dir, 'log.txt'), 'wt')
        return HumanOutputFormat(log_file)
    elif format == 'json':
        json_file = open(osp.join(ev_dir, 'progress.json'), 'wt')
        return JSONOutputFormat(json_file)
    elif format == 'tensorboard':
        return TensorBoardOutputFormat(osp.join(ev_dir, 'tb'))
    else:
        raise ValueError('Unknown format specified: %s' % (format,))

# ================================================================
# API
# ================================================================ 
Example 34
Project: seq2seq   Author: google   File: generate_toy_data.py View Source Project 6 votes vote down vote up
def main():
  """Main function"""

  if ARGS.type == "copy":
    generate_fn = make_copy
  elif ARGS.type == "reverse":
    generate_fn = make_reverse

  # Generate dataset
  examples = list(generate_fn(ARGS.num_examples, ARGS.min_len, ARGS.max_len))
  try:
    os.makedirs(ARGS.output_dir)
  except OSError:
    if not os.path.isdir(ARGS.output_dir):
      raise

  # Write train data
  train_sources, train_targets = zip(*examples)
  write_parallel_text(train_sources, train_targets, ARGS.output_dir) 
Example 35
Project: zipline-chinese   Author: zhanghan1990   File: core.py View Source Project 6 votes vote down vote up
def add_security_data(adds, deletes):
    if not hasattr(security_list, 'using_copy'):
        raise Exception('add_security_data must be used within '
                        'security_list_copy context')
    directory = os.path.join(
        security_list.SECURITY_LISTS_DIR,
        "leveraged_etf_list/20150127/20150125"
    )
    if not os.path.exists(directory):
        os.makedirs(directory)
    del_path = os.path.join(directory, "delete")
    with open(del_path, 'w') as f:
        for sym in deletes:
            f.write(sym)
            f.write('\n')
    add_path = os.path.join(directory, "add")
    with open(add_path, 'w') as f:
        for sym in adds:
            f.write(sym)
            f.write('\n') 
Example 36
Project: zipline-chinese   Author: zhanghan1990   File: generate_new_sample_saved_state.py View Source Project 6 votes vote down vote up
def write_state_to_disk(cls, state, emission_rate=None):
    state_dir = cls.__module__ + '.' + cls.__name__

    full_dir = base_state_dir + '/' + state_dir

    if not os.path.exists(full_dir):
        os.makedirs(full_dir)

    if emission_rate is not None:
        name = 'State_Version_' + emission_rate + \
            str(state['obj_state'][VERSION_LABEL])
    else:
        name = 'State_Version_' + str(state['obj_state'][VERSION_LABEL])

    full_path = full_dir + '/' + name

    f = open(full_path, 'w')

    pickle.dump(state, f)

    f.close() 
Example 37
Project: okta-awscli   Author: jmhale   File: aws_auth.py View Source Project 6 votes vote down vote up
def write_sts_token(self, profile, access_key_id, secret_access_key, session_token):
        """ Writes STS auth information to credentials file """
        region = 'us-east-1'
        output = 'json'
        if not os.path.exists(self.creds_dir):
            os.makedirs(self.creds_dir)
        config = RawConfigParser()

        if os.path.isfile(self.creds_file):
            config.read(self.creds_file)

        if not config.has_section(profile):
            config.add_section(profile)

        config.set(profile, 'output', output)
        config.set(profile, 'region', region)
        config.set(profile, 'aws_access_key_id', access_key_id)
        config.set(profile, 'aws_secret_access_key', secret_access_key)
        config.set(profile, 'aws_session_token', session_token)

        with open(self.creds_file, 'w+') as configfile:
            config.write(configfile)
        print("Temporary credentials written to profile: %s" % profile)
        print("Invoke using: aws --profile %s <service> <command>" % profile) 
Example 38
Project: cellranger   Author: 10XGenomics   File: supernova.py View Source Project 6 votes vote down vote up
def write_stage_alerts(stage, path, alerts_file="alerts.list"):
    alerts = load_alerts()
    out_file = os.path.join(path, alerts_file)
    if not os.path.exists(path):
        os.makedirs(path)
    out_handle = open(out_file, "w")
    keys = ["metric", "threshold", "compare", "action", "message"]
    if not alerts.has_key(stage):
        martian.throw("No alerts found for stage %s" % stage)
    for alert in alerts[stage]:
        out_handle.write("#\n")
        out_handle.write(alert["metric"]+"\n")
        out_handle.write(str(alert["threshold"])+"\n")
        out_handle.write(alert["compare"]+"\n")
        out_handle.write(alert["action"]+"\n")
        out_handle.write(alert["message"]+"\n") 
    out_handle.close() 
Example 39
Project: acbs   Author: AOSC-Dev   File: acbs-build.py View Source Project 6 votes vote down vote up
def init_env(tree=['default']):
    dump_loc = '/var/cache/acbs/tarballs/'
    tmp_loc = '/var/cache/acbs/build/'
    print("----- Welcome to ACBS - %s -----" % (acbs_version))
    try:
        if not os.path.isdir(dump_loc):
            os.makedirs(dump_loc)
        if not os.path.isdir(tmp_loc):
            os.makedirs(tmp_loc)
    except:
        raise IOError('\033[93mFailed to make work directory\033[0m!')
    if os.path.exists('/etc/acbs_forest.conf'):
        tree_loc = parse_acbs_conf(tree[0])
        if tree_loc is not None:
            os.chdir(tree_loc)
        else:
            sys.exit(1)
    else:
        if not write_acbs_conf():
            sys.exit(1)
    return 
Example 40
Project: yt-browser   Author: juanfgs   File: configuration.py View Source Project 6 votes vote down vote up
def __init__(self):
        self.config_dir = os.path.expanduser("~/.config/ytbrowser/")
        self.defaults['format'] = "mkv"
        self.defaults['quality'] = "bestvideo"
        self.defaults['preferredcodec'] = "mp3"
        self.defaults['preferredquality'] = 192
        self.defaults['developerKey'] = "AIzaSyDFuK00HWV0fd1VMb17R8GghRVf_iQx9uk"
        self.defaults['apiServiceName'] = "youtube"
        self.defaults['apiVersion'] = "v3"

        if not os.path.exists(self.config_dir):
            os.makedirs(self.config_dir)

        if not os.path.exists(self.config_dir + "config.yml"):
            open(self.config_dir + "config.yml", "a").close()

        with open(self.config_dir + "config.yml", 'r') as ymlfile:
                self.user_settings = yaml.load(ymlfile)

        if self.user_settings is None:
            self.user_settings = {} 
Example 41
Project: pycos   Author: pgiri   File: netpycos.py View Source Project 5 votes vote down vote up
def dest_path(self, path):
        path = os.path.normpath(path)
        if not path.startswith(self.__dest_path_prefix):
            path = os.path.join(self.__dest_path_prefix,
                                os.path.splitdrive(path)[1].lstrip(os.sep))
        try:
            os.makedirs(path)
        except OSError as exc:
            if exc.errno != errno.EEXIST:
                raise
        self.__dest_path = path 
Example 42
Project: pycos   Author: pgiri   File: netpycos.py View Source Project 5 votes vote down vote up
def dest_path(self, path):
        path = os.path.normpath(path)
        if not path.startswith(self.__dest_path_prefix):
            path = os.path.join(self.__dest_path_prefix,
                                os.path.splitdrive(path)[1].lstrip(os.sep))
        try:
            os.makedirs(path)
        except OSError as exc:
            if exc.errno != errno.EEXIST:
                raise
        self.__dest_path = path 
Example 43
Project: Cat-Segmentation   Author: ardamavi   File: get_dataset.py View Source Project 5 votes vote down vote up
def get_dataset(dataset_path='Data/Train_Data'):
    # Getting all data from data path:
    try:
        X = np.load('Data/npy_train_data/X.npy')
        Y = np.load('Data/npy_train_data/Y.npy')
    except:
        inputs_path = dataset_path+'/input'
        images = listdir(inputs_path) # Geting images
        X = []
        Y = []
        for img in images:
            img_path = inputs_path+'/'+img

            x_img = get_img(img_path).astype('float32').reshape(64, 64, 3)
            x_img /= 255.

            y_img = get_img(img_path.replace('input/', 'mask/mask_')).astype('float32').reshape(64, 64, 1)
            y_img /= 255.

            X.append(x_img)
            Y.append(y_img)
        X = np.array(X)
        Y = np.array(Y)
        # Create dateset:
        if not os.path.exists('Data/npy_train_data/'):
            os.makedirs('Data/npy_train_data/')
        np.save('Data/npy_train_data/X.npy', X)
        np.save('Data/npy_train_data/Y.npy', Y)
    X, X_test, Y, Y_test = train_test_split(X, Y, test_size=0.1, random_state=42)
    return X, X_test, Y, Y_test 
Example 44
Project: Cat-Segmentation   Author: ardamavi   File: train.py View Source Project 5 votes vote down vote up
def train_model(model, X, X_test, Y, Y_test):
    if not os.path.exists('Data/Checkpoints/'):
        os.makedirs('Data/Checkpoints/')
    checkpoints = []
    checkpoints.append(ModelCheckpoint('Data/Checkpoints/best_weights.h5', monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=True, mode='auto', period=1))
    checkpoints.append(TensorBoard(log_dir='Data/Checkpoints/./logs', histogram_freq=0, write_graph=True, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None))

    model.fit(X, Y, batch_size=batch_size, epochs=epochs, validation_data=(X_test, Y_test), shuffle=True, callbacks=checkpoints)

    return model 
Example 45
Project: Cat-Segmentation   Author: ardamavi   File: get_model.py View Source Project 5 votes vote down vote up
def save_model(model):
    if not os.path.exists('Data/Model/'):
        os.makedirs('Data/Model/')
    model_json = model.to_json()
    with open("Data/Model/model.json", "w") as model_file:
        model_file.write(model_json)
    # serialize weights to HDF5
    model.save_weights("Data/Model/weights.h5")
    print('Model and weights saved')
    return 
Example 46
Project: charm-plumgrid-gateway   Author: openstack   File: charm_helpers_sync.py View Source Project 5 votes vote down vote up
def sync_pyfile(src, dest):
    src = src + '.py'
    src_dir = os.path.dirname(src)
    logging.info('Syncing pyfile: %s -> %s.' % (src, dest))
    if not os.path.exists(dest):
        os.makedirs(dest)
    shutil.copy(src, dest)
    if os.path.isfile(os.path.join(src_dir, '__init__.py')):
        shutil.copy(os.path.join(src_dir, '__init__.py'),
                    dest)
    ensure_init(dest) 
Example 47
Project: AutoML5   Author: djajetic   File: data_manager.py View Source Project 5 votes vote down vote up
def __init__(self, basename, input_dir, verbose=False, replace_missing=True, filter_features=False):
		'''Constructor'''
		self.use_pickle = False # Turn this to true to save data as pickle (inefficient)
		self.basename = basename
   		if basename in input_dir:
			self.input_dir = input_dir 
   		else:
			self.input_dir = input_dir + "/" + basename + "/"   
		if self.use_pickle:
			if os.path.exists ("tmp"):
				self.tmp_dir = "tmp"
			elif os.path.exists ("../tmp"):
				self.tmp_dir = "../tmp" 
			else:
				os.makedirs("tmp")
				self.tmp_dir = "tmp"
		info_file = os.path.join (self.input_dir, basename + '_public.info')
		self.info = {}
		self.getInfo (info_file)
         	self.feat_type = self.loadType (os.path.join(self.input_dir, basename + '_feat.type'), verbose=verbose)
		self.data = {}  
		Xtr = self.loadData (os.path.join(self.input_dir, basename + '_train.data'), verbose=verbose, replace_missing=replace_missing)
		Ytr = self.loadLabel (os.path.join(self.input_dir, basename + '_train.solution'), verbose=verbose)
		Xva = self.loadData (os.path.join(self.input_dir, basename + '_valid.data'), verbose=verbose, replace_missing=replace_missing)
		Xte = self.loadData (os.path.join(self.input_dir, basename + '_test.data'), verbose=verbose, replace_missing=replace_missing)
           # Normally, feature selection should be done as part of a pipeline.
           # However, here we do it as a preprocessing for efficiency reason
		idx=[]
		if filter_features: # add hoc feature selection, for the example...
			fn = min(Xtr.shape[1], 1000)       
			idx = data_converter.tp_filter(Xtr, Ytr, feat_num=fn, verbose=verbose)
			Xtr = Xtr[:,idx]
			Xva = Xva[:,idx]
			Xte = Xte[:,idx]  
		self.feat_idx = np.array(idx).ravel()
		self.data['X_train'] = Xtr
		self.data['Y_train'] = Ytr
		self.data['X_valid'] = Xva
		self.data['X_test'] = Xte 
Example 48
Project: AutoML5   Author: djajetic   File: libscores.py View Source Project 5 votes vote down vote up
def mkdir(d):
    if not os.path.exists(d):
        os.makedirs(d) 
Example 49
Project: AutoML5   Author: djajetic   File: data_io.py View Source Project 5 votes vote down vote up
def mkdir(d):
    ''' Create a new directory'''
    if not os.path.exists(d):
        os.makedirs(d) 
Example 50
Project: hesperides-cli   Author: voyages-sncf-technologies   File: configfile.py View Source Project 5 votes vote down vote up
def _create_home(self):
        if not os.path.isdir(self._HOME + '/' + self._CONFIG_DIR):
            os.makedirs(self._HOME + '/' + self._CONFIG_DIR)
        with os.fdopen(os.open(self._HOME + '/' + self._CONFIG_DIR + '/' + self._CONFIG_FILE_NAME,
                               os.O_WRONLY | os.O_CREAT, 0o600), 'w'):
            pass
        with os.fdopen(os.open(self._HOME + '/' + self._CONFIG_DIR + '/' + self._CREDENTIALS_FILE_NAME,
                               os.O_WRONLY | os.O_CREAT, 0o600), 'w'):
            pass