Python os.makedirs() Examples

def downloadDemo(which):
    try:
        downloadDir = tempfile.mkdtemp()
        archivePath = "{}/svviz-data.zip".format(downloadDir)

        # logging.info("Downloading...")
        downloadWithProgress("http://svviz.github.io/svviz/assets/examples/{}.zip".format(which), archivePath)
        
        logging.info("Decompressing...")
        archive = zipfile.ZipFile(archivePath)
        archive.extractall("{}".format(downloadDir))

        if not os.path.exists("svviz-examples"):
            os.makedirs("svviz-examples/")

        shutil.move("{temp}/{which}".format(temp=downloadDir, which=which), "svviz-examples/")
    except Exception as e:
        print("error downloading and decompressing example data: {}".format(e))
        return False

    if not os.path.exists("svviz-examples"):
        print("error finding example data after download and decompression")
        return False
    return True 

def create_oa_folders(cls, type, date):

        # create date and ingest summary folder structure if they don't' exist.
        root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
        data_type_folder = "{0}/data/{1}/{2}"
        if not os.path.isdir(data_type_folder.format(root_path, type, date)): os.makedirs(
            data_type_folder.format(root_path, type, date))
        if not os.path.isdir(data_type_folder.format(root_path, type, "ingest_summary")): os.makedirs(
            data_type_folder.format(root_path, type, "ingest_summary"))

        # create ipynb folders.
        ipynb_folder = "{0}/ipynb/{1}/{2}".format(root_path, type, date)
        if not os.path.isdir(ipynb_folder): os.makedirs(ipynb_folder)

        # retun path to folders.
        data_path = data_type_folder.format(root_path, type, date)
        ingest_path = data_type_folder.format(root_path, type, "ingest_summary")
        return data_path, ingest_path, ipynb_folder 

def ensure_session_manager_plugin():
    session_manager_dir = os.path.join(config.user_config_dir, "bin")
    PATH = os.environ.get("PATH", "") + ":" + session_manager_dir
    if shutil.which("session-manager-plugin", path=PATH):
        subprocess.check_call(["session-manager-plugin"], env=dict(os.environ, PATH=PATH))
    else:
        os.makedirs(session_manager_dir, exist_ok=True)
        target_path = os.path.join(session_manager_dir, "session-manager-plugin")
        if platform.system() == "Darwin":
            download_session_manager_plugin_macos(target_path=target_path)
        elif platform.linux_distribution()[0] == "Ubuntu":
            download_session_manager_plugin_linux(target_path=target_path)
        else:
            download_session_manager_plugin_linux(target_path=target_path, pkg_format="rpm")
        os.chmod(target_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
        subprocess.check_call(["session-manager-plugin"], env=dict(os.environ, PATH=PATH))
    return shutil.which("session-manager-plugin", path=PATH) 

def save_to_path(self, filepath):
        """Save retrieved data to file at ``filepath``.

        .. versionadded: 1.9.6

        :param filepath: Path to save retrieved data.

        """
        filepath = os.path.abspath(filepath)
        dirname = os.path.dirname(filepath)
        if not os.path.exists(dirname):
            os.makedirs(dirname)

        self.stream = True

        with open(filepath, 'wb') as fileobj:
            for data in self.iter_content():
                fileobj.write(data) 

def save_model_all(model, save_dir, model_name, epoch):
    """
    :param model:  nn model
    :param save_dir: save model direction
    :param model_name:  model name
    :param epoch:  epoch
    :return:  None
    """
    if not os.path.isdir(save_dir):
        os.makedirs(save_dir)
    save_prefix = os.path.join(save_dir, model_name)
    save_path = '{}_epoch_{}.pt'.format(save_prefix, epoch)
    print("save all model to {}".format(save_path))
    output = open(save_path, mode="wb")
    torch.save(model.state_dict(), output)
    # torch.save(model.state_dict(), save_path)
    output.close() 

def mkdir_p(path):
    if not os.path.exists(path):
        print('creating directory: ' + path)
        os.makedirs(path)


# Remove files and/or directories recursively 

def initialize():
    global config, parser
    from .util.printing import BOLD, RED, ENDC
    config = AegeaConfig(__name__, use_yaml=True, save_on_exit=False)
    if not os.path.exists(config.config_files[2]):
        config_dir = os.path.dirname(os.path.abspath(config.config_files[2]))
        try:
            os.makedirs(config_dir)
        except OSError as e:
            if not (e.errno == errno.EEXIST and os.path.isdir(config_dir)):
                raise
        shutil.copy(os.path.join(os.path.dirname(__file__), "user_config.yml"), config.config_files[2])
        logger.info("Wrote new config file %s with default values", config.config_files[2])
        config = AegeaConfig(__name__, use_yaml=True, save_on_exit=False)

    parser = argparse.ArgumentParser(
        description="{}: {}".format(BOLD() + RED() + __name__.capitalize() + ENDC(), fill(__doc__.strip())),
        formatter_class=AegeaHelpFormatter
    )
    parser.add_argument("--version", action="version", version="%(prog)s {}\n{} {}\n{}".format(
        __version__,
        platform.python_implementation(),
        platform.python_version(),
        platform.platform()
    ))

    def help(args):
        parser.print_help()
    register_parser(help) 

def _remote_init(working_dir):
    global pickle
    import pickle
    import sys
    import shutil
    import os
    if not os.path.isdir(working_dir):
        os.mkdir(working_dir)
    sys.path.append(working_dir)
    shutil.move("_common.py", working_dir)
    shutil.move("_gdb.py", working_dir)
    shutil.move("cmds.gdb", working_dir)
    # setup CERT exploitable
    exp_lib_dir = os.path.join(working_dir, "exploitable", "lib")
    os.makedirs(exp_lib_dir)
    shutil.move("exploitable.py", os.path.join(working_dir, "exploitable"))
    shutil.move("__init__.py", exp_lib_dir)
    shutil.move("analyzers.py", exp_lib_dir)
    shutil.move("classifier.py", exp_lib_dir)
    shutil.move("elf.py", exp_lib_dir)
    shutil.move("gdb_wrapper.py", exp_lib_dir)
    shutil.move("rules.py", exp_lib_dir)
    shutil.move("tools.py", exp_lib_dir)
    shutil.move("versions.py", exp_lib_dir)
    os.chdir(working_dir)
    global _common
    global _gdb
    import _common
    import _gdb 

def savepb(self):
		"""
		Create a standalone const graph def that 
		C++	can load and run.
		"""
		darknet_pb = self.to_darknet()
		flags_pb = self.FLAGS
		flags_pb.verbalise = False
		
		flags_pb.train = False
		# rebuild another tfnet. all const.
		tfnet_pb = TFNet(flags_pb, darknet_pb)		
		tfnet_pb.sess = tf.Session(graph = tfnet_pb.graph)
		# tfnet_pb.predict() # uncomment for unit testing
		name = 'built_graph/{}.pb'.format(self.meta['name'])
		os.makedirs(os.path.dirname(name), exist_ok=True)
		#Save dump of everything in meta
		with open('built_graph/{}.meta'.format(self.meta['name']), 'w') as fp:
			json.dump(self.meta, fp)
		self.say('Saving const graph def to {}'.format(name))
		graph_def = tfnet_pb.sess.graph_def
		tf.train.write_graph(graph_def,'./', name, False) 

def fetch_attacks_data(self):
    """Initializes data necessary to execute attacks.

    This method could be called multiple times, only first call does
    initialization, subsequent calls are noop.
    """
    if self.attacks_data_initialized:
      return
    # init data from datastore
    self.submissions.init_from_datastore()
    self.dataset_batches.init_from_datastore()
    self.adv_batches.init_from_datastore()
    # copy dataset locally
    if not os.path.exists(LOCAL_DATASET_DIR):
      os.makedirs(LOCAL_DATASET_DIR)
    eval_lib.download_dataset(self.storage_client, self.dataset_batches,
                              LOCAL_DATASET_DIR,
                              os.path.join(LOCAL_DATASET_COPY,
                                           self.dataset_name, 'images'))
    # download dataset metadata
    self.read_dataset_metadata()
    # mark as initialized
    self.attacks_data_initialized = True 

def writeDir_r(self, det_dir, dire, pp, r, all_type):
        #gen.log "writeDir_r:(%s)"%(det_dir)
        dirs = self.readDirItems(dire.locExtent, dire.lenData)
        for d in dirs:
            if not d.fIdentifier in [".", ".."]:
                if (pp != None) and (pp.search(d.fIdentifier) == None):
                    match = False
                else:
                    match = True
                #gen.log "mathing %s, %s, (%x)"%(match, d.fIdentifier, d.fFlag)
                p = det_dir + "/" + d.fIdentifier
                if d.fFlag & 0x02 == 0x02:
                    if not os.path.exists(p):
                        os.makedirs(p, 0o777)
                    if r:
                        if match:
                            self.writeDir_r(p, d, None, r, all_type) # Don't need to match subdirectory.
                        else:
                            self.writeDir_r(p, d, pp, r, all_type)
                elif match:
                    self.writeFile(d, p, all_type)
            # if not d.fIdentifier end #
        # for d in dirs end # 

def plot_examples(data_loader, model, epoch, plotter, ind = [0, 10, 20]):

    # switch to evaluate mode
    model.eval()

    for i, (g, h, e, target) in enumerate(data_loader):
        if i in ind:
            subfolder_path = 'batch_' + str(i) + '_t_' + str(int(target[0][0])) + '/epoch_' + str(epoch) + '/'
            if not os.path.isdir(args.plotPath + subfolder_path):
                os.makedirs(args.plotPath + subfolder_path)

            num_nodes = torch.sum(torch.sum(torch.abs(h[0, :, :]), 1) > 0)
            am = g[0, 0:num_nodes, 0:num_nodes].numpy()
            pos = h[0, 0:num_nodes, :].numpy()

            plotter.plot_graph(am, position=pos, fig_name=subfolder_path+str(i) + '_input.png')

            # Prepare input data
            if args.cuda:
                g, h, e, target = g.cuda(), h.cuda(), e.cuda(), target.cuda()
            g, h, e, target = Variable(g), Variable(h), Variable(e), Variable(target)

            # Compute output
            model(g, h, e, lambda cls, id: plotter.plot_graph(am, position=pos, cls=cls,
                                                          fig_name=subfolder_path+ id)) 

def get_models_overall(exp_name, rbp):
    print("RBP: " + rbp)

    out_h5 = "{rbp}/model_files/model.h5".format(rbp=rbp)
    os.makedirs(os.path.dirname(out_h5), exist_ok=True)

    trials = CMongoTrials(DB_NAME, exp_name + "_" + rbp, ip=HOST)

    # no trials yet - return None
    if trials.n_ok() == 0:
        trials = CMongoTrials(DB_NAME[:-2], exp_name + "_" + rbp, ip=HOST)
        if trials.n_ok() == 0:
            raise Exception("No trials")
    print("N trials: {0}".format(trials.n_ok()))

    # get best trial parameters
    tid = trials.best_trial_tid()
    model_path = trials.get_trial(tid)["result"]["path"]["model"]
    copyfile(model_path, out_h5) 

def save_best_model(model, save_dir, model_name, best_eval):
    """
    :param model:  nn model
    :param save_dir:  save model direction
    :param model_name:  model name
    :param best_eval:  eval best
    :return:  None
    """
    if best_eval.current_dev_score >= best_eval.best_dev_score:
        if not os.path.isdir(save_dir): os.makedirs(save_dir)
        model_name = "{}.pt".format(model_name)
        save_path = os.path.join(save_dir, model_name)
        print("save best model to {}".format(save_path))
        # if os.path.exists(save_path):  os.remove(save_path)
        output = open(save_path, mode="wb")
        torch.save(model.state_dict(), output)
        # torch.save(model.state_dict(), save_path)
        output.close()
        best_eval.early_current_patience = 0


# adjust lr 

def cache(self, dataset, cache, wait):
        """ Cache the given dataset if cache is enabled. Eventually waits for
        cache to be available (useful if another process is already computing
        cache) if provided wait flag is True.

        :param dataset: Dataset to be cached if cache is required.
        :param cache: Path of cache directory to be used, None if no cache.
        :param wait: If caching is enabled, True is cache should be waited.
        :returns: Cached dataset if needed, original dataset otherwise.
        """
        if cache is not None:
            if wait:
                while not exists(f'{cache}.index'):
                    get_logger().info(
                        'Cache not available, wait %s',
                        self.WAIT_PERIOD)
                    time.sleep(self.WAIT_PERIOD)
            cache_path = os.path.split(cache)[0]
            os.makedirs(cache_path, exist_ok=True)
            return dataset.cache(cache)
        return dataset 

def download_files(self, urls, env, headers=None, dir=None):

        if dir is None:
            dir = env.get('cache-dir')
        
        dest_directory = os.path.join(dir, "tmp_" + str(uuid.uuid4()))
        
        if not os.path.exists(dest_directory):
            os.makedirs(dest_directory)

        for data_url in urls:
            if isinstance(data_url, tuple):
                data_url, download_file = data_url
            else:
                download_file = data_url.split('/')[-1]

            download_path = os.path.join(dest_directory, download_file)

            if headers:
                opener = urllib.request.build_opener()
                opener.addheaders = headers
                urllib.request.install_opener(opener)

            try:
                urllib.request.urlretrieve(data_url, filename=download_path, 
                                           reporthook=self._report_hook(download_file), data=None)
            except Exception as e:
                raise VergeMLError("Could not download {}: {}".format(data_url, e))
            finally:
                if headers:
                    urllib.request.install_opener(urllib.request.build_opener())

        return dest_directory 

def _get_cachedir():
    cachedir = os.path.join(util.get_tempdir(), 'cache')
    if not os.path.exists(cachedir):
        LOG.debug('Creating cache directory: ' + cachedir)
    if not os.path.exists(cachedir):
        os.makedirs(cachedir)
    return cachedir 

def generate_config(username=None, password=None, servicename="MLB.tv"):
        """Creates config file from template + user prompts."""
        script_name = os.path.splitext(os.path.basename(sys.argv[0]))[0]
        # use the script name minus any extension for the config directory
        config_dir = None
        config_dir = os.path.join(Config.config_dir_roots[1], script_name)
        if not os.path.exists(config_dir):
            print("Creating config directory: {}".format(config_dir))
            os.makedirs(config_dir)
        config_file = os.path.join(config_dir, 'config')
        if os.path.exists(config_file):
            print("Aborting: The config file already exists at '{}'".format(config_file))
            return False

        # copy the template config file
        print("Generating basic config file at: {}".format(config_dir))
        current_dir = os.path.dirname(inspect.getfile(inspect.currentframe()))
        template_config_path = os.path.abspath(os.path.join(current_dir, '../../..', 'config.template'))
        if not os.path.exists(template_config_path):
            print("Could not find template config file [expected at: {}]".format(template_config_path))
            return False

        if username is None:
            username = input('Enter {} username: '.format(servicename))
        if password is None:
            password = input('Enter {} password: '.format(servicename))

        with open(template_config_path, 'r') as infile, open(config_file, 'w') as outfile:
            for line in infile:
                if line.startswith('# username='):
                    outfile.write("username={}\n".format(username))
                elif line.startswith('# password='):
                    outfile.write("password={}\n".format(password))
                else:
                    outfile.write(line)
        print("Finished creating config file: {}".format(config_file))
        print("You may want to edit it now to set up favourites, etc.")
        return True 

def download(x):
    count=0
    article_link=x.find_elements_by_class_name('home__timeline__item')
    logfp=open(log_file,'a')
    for l in article_link:
        try:
            time=l.find_element_by_class_name('timestamp').text.split(' ')[0]
            time=sdtime(time)
            l.find_element_by_xpath('h3/a').click()
            t=l.find_element_by_xpath('h3/a').text
            title,flag=clean_invalid(t)
            if flag:
                logfp.writelines(t)
                logfp.write('\n')
            windows = x.window_handles
            x.switch_to.window(windows[-1])
            txt=x.find_element_by_xpath('//div[@class="article__bd__detail"]').text
                
            if not os.path.exists(work_space+'/article/'+time+'/'):
                os.makedirs(work_space+'/article/'+time+'/')
            txt_filename=work_space+'/article/'+time+'/'+title+'.txt'
            fp=open(txt_filename,'w',encoding='utf-8')
            fp.writelines(txt)
            fp.close()
            x.close()
            count+=1
            x.switch_to.window(windows[0])
        except:
            windows = x.window_handles
            x.switch_to.window(windows[0])
            continue
    logfp.close()
    print('downloaded %d articles.' % count) 

def makedirs(name, mode=0o777, exist_ok=False):
        try:
            os.makedirs(name, mode)
        except OSError as e:
            if not (exist_ok and e.errno == errno.EEXIST and os.path.isdir(name)):
                raise 

def download_from_github(arch):
    templates = str(Path.home()) + "/.config/arm_now/templates/"
    os.makedirs(templates)
    filename = arch + ".tar.xz"
    URL = "https://github.com/nongiach/arm_now_templates/raw/master/"
    download(URL + filename, templates + filename, Config.DOWNLOAD_CACHE_DIR) 

def do_offline():
    URL = "https://github.com/nongiach/arm_now_templates/archive/master.zip"
    templates = str(Path.home()) + "/.config/arm_now/templates/"
    master_zip = str(Path.home()) + "/.config/arm_now/templates/master.zip"
    os.makedirs(templates)
    # download_from_github(arch)
    download(URL, master_zip, Config.DOWNLOAD_CACHE_DIR)
    os.chdir(templates)
    check_call("unzip master.zip", shell=True)
    check_call("mv arm_now_templates-master/* .", shell=True)
    check_call("rm -rf arm_now_templates-master/ README.md master.zip", shell=True) 

def _save_files(self, data, dtype_out_time):
        """Save the data to netcdf files in direc_out."""
        path = self.path_out[dtype_out_time]
        if not os.path.isdir(self.dir_out):
            os.makedirs(self.dir_out)
        if 'reg' in dtype_out_time:
            try:
                reg_data = xr.open_dataset(path)
            except (EOFError, RuntimeError, IOError):
                reg_data = xr.Dataset()
            reg_data.update(data)
            data_out = reg_data
        else:
            data_out = data
        if isinstance(data_out, xr.DataArray):
            data_out = xr.Dataset({self.name: data_out})
        data_out.to_netcdf(path, engine='netcdf4') 

def _create(self, dirpath):
        """Create directory `dirpath` if it doesn't exist.

        :param dirpath: path to directory
        :type dirpath: ``unicode``
        :returns: ``dirpath`` argument
        :rtype: ``unicode``

        """
        if not os.path.exists(dirpath):
            os.makedirs(dirpath)
        return dirpath 

def __init__(self, network, imdb, roidb, valroidb, output_dir, tbdir, pretrained_model=None, wsddn_premodel=None):
    self.net = network
    self.imdb = imdb
    self.roidb = roidb
    self.valroidb = valroidb
    self.output_dir = output_dir
    self.tbdir = tbdir
    # Simply put '_val' at the end to save the summaries from the validation set
    self.tbvaldir = tbdir + '_val'
    if not os.path.exists(self.tbvaldir):
      os.makedirs(self.tbvaldir)
    self.pretrained_model = pretrained_model
    self.wsddn_premodel = wsddn_premodel 

def snapshot(self, iter):
    net = self.net

    if not os.path.exists(self.output_dir):
      os.makedirs(self.output_dir)

    # Store the model snapshot
    filename = cfg.TRAIN.SNAPSHOT_PREFIX + '_iter_{:d}'.format(iter) + '.pth'
    filename = os.path.join(self.output_dir, filename)
    torch.save(self.net.state_dict(), filename)
    print('Wrote snapshot to: {:s}'.format(filename))
    
    
    if iter % 10000 == 0:
        shutil.copyfile(filename, filename + '.{:d}_cache'.format(iter))
    
    # Also store some meta information, random state, etc.
    nfilename = cfg.TRAIN.SNAPSHOT_PREFIX + '_iter_{:d}'.format(iter) + '.pkl'
    nfilename = os.path.join(self.output_dir, nfilename)
    # current state of numpy random
    st0 = np.random.get_state()
    # current position in the database
    cur = self.data_layer._cur
    # current shuffled indexes of the database
    perm = self.data_layer._perm
    # current position in the validation database
    cur_val = self.data_layer_val._cur
    # current shuffled indexes of the validation database
    perm_val = self.data_layer_val._perm

    # Dump the meta info
    with open(nfilename, 'wb') as fid:
      pickle.dump(st0, fid, pickle.HIGHEST_PROTOCOL)
      pickle.dump(cur, fid, pickle.HIGHEST_PROTOCOL)
      pickle.dump(perm, fid, pickle.HIGHEST_PROTOCOL)
      pickle.dump(cur_val, fid, pickle.HIGHEST_PROTOCOL)
      pickle.dump(perm_val, fid, pickle.HIGHEST_PROTOCOL)
      pickle.dump(iter, fid, pickle.HIGHEST_PROTOCOL)

    return filename, nfilename 

def get_output_dir(imdb, weights_filename):
  """Return the directory where experimental artifacts are placed.
  If the directory does not exist, it is created.

  A canonical path is built using the name from an imdb and a network
  (if not None).
  """
  outdir = osp.abspath(osp.join(__C.ROOT_DIR, 'output', __C.EXP_DIR, imdb.name))
  if weights_filename is None:
    weights_filename = 'default'
  outdir = osp.join(outdir, weights_filename)
  if not os.path.exists(outdir):
    os.makedirs(outdir)
  return outdir 

def get_output_tb_dir(imdb, weights_filename):
  """Return the directory where tensorflow summaries are placed.
  If the directory does not exist, it is created.

  A canonical path is built using the name from an imdb and a network
  (if not None).
  """
  outdir = osp.abspath(osp.join(__C.ROOT_DIR, 'tensorboard', __C.EXP_DIR, imdb.name))
  if weights_filename is None:
    weights_filename = 'default'
  outdir = osp.join(outdir, weights_filename)
  if not os.path.exists(outdir):
    os.makedirs(outdir)
  return outdir 

def cache_path(self):
    cache_path = osp.abspath(osp.join(cfg.DATA_DIR, 'cache'))
    if not os.path.exists(cache_path):
      os.makedirs(cache_path)
    return cache_path 

def preparation():
    # set model save path
    if args.if_load_from_checkpoint:
        timestamp = args.checkpoint_name
    else:
        timestamp = str(int(time.time()))
        print("create new model save path: %s" % timestamp)
    args.current_save_path = 'save/%s/' % timestamp
    args.log_file = args.current_save_path + time.strftime("log_%Y_%m_%d_%H_%M_%S.txt", time.localtime())
    args.output_file = args.current_save_path + time.strftime("output_%Y_%m_%d_%H_%M_%S.txt", time.localtime())
    print("create log file at path: %s" % args.log_file)

    if os.path.exists(args.current_save_path):
        add_log("Load checkpoint model from Path: %s" % args.current_save_path)
    else:
        os.makedirs(args.current_save_path)
        add_log("Path: %s is created" % args.current_save_path)

    # set task type
    if args.task == 'yelp':
        args.data_path = '../../data/yelp/processed_files/'
    elif args.task == 'amazon':
        args.data_path = '../../data/amazon/processed_files/'
    elif args.task == 'imagecaption':
        pass
    else:
        raise TypeError('Wrong task type!')

    # prepare data
    args.id_to_word, args.vocab_size, \
    args.train_file_list, args.train_label_list = prepare_data(
        data_path=args.data_path, max_num=args.word_dict_max_num, task_type=args.task
    )
    return