Python shutil.rmtree() Examples

def test_multiprocessing_download_successful():
    """ test download with multiprocessing """
    tmp = tempfile.mkdtemp()
    tmpfile = os.path.join(tmp, 'README.md')
    process_list = []
    # test it with 10 processes
    for i in range(10):
        process_list.append(mp.Process(
            target=_download_successful, args=(tmpfile,)))
        process_list[i].start()
    for i in range(10):
        process_list[i].join()
    assert os.path.getsize(tmpfile) > 100, os.path.getsize(tmpfile)
    # check only one file we want left
    pattern = os.path.join(tmp, 'README.md*')
    assert len(glob.glob(pattern)) == 1, glob.glob(pattern)
    # delete temp dir
    shutil.rmtree(tmp) 

def test_extraction():
    import shutil

    src = 'c:/Users/shinj/Downloads/clonezilla-live-2.5.2-31-amd64.iso'
    tmp_dir = 'c:/Users/shinj/Documents/tmp'
    for subdir, pattern in [
            ('single_string', 'EFI/'),
            ('single_list', ['EFI/']),
            ('multi', ['EFI/', 'syslinux/']),
            ('all', None) ]:
        dest_dir = os.path.join(tmp_dir, subdir)
        if os.path.exists(dest_dir):
            shutil.rmtree(dest_dir)
        os.mkdir(dest_dir)
        args = [src, dest_dir]
        if pattern is not None:
            args.append(pattern)
        print ('Calling extract_iso(%s)' % args)
        extract_iso(*args) 

def save_dictionary(config):
    """
    :param config: config
    :return:
    """
    if config.save_dict is True:
        if os.path.exists(config.dict_directory):
            shutil.rmtree(config.dict_directory)
        if not os.path.isdir(config.dict_directory):
            os.makedirs(config.dict_directory)

        config.word_dict_path = "/".join([config.dict_directory, config.word_dict])
        config.label_dict_path = "/".join([config.dict_directory, config.label_dict])
        print("word_dict_path : {}".format(config.word_dict_path))
        print("label_dict_path : {}".format(config.label_dict_path))
        save_dict2file(config.create_alphabet.word_alphabet.words2id, config.word_dict_path)
        save_dict2file(config.create_alphabet.label_alphabet.words2id, config.label_dict_path)
        # copy to mulu
        print("copy dictionary to {}".format(config.save_dir))
        shutil.copytree(config.dict_directory, "/".join([config.save_dir, config.dict_directory]))


# load data / create alphabet / create iterator 

def _delete_directory_contents(self, dirpath, filter_func):
        """Delete all files in a directory.

        :param dirpath: path to directory to clear
        :type dirpath: ``unicode`` or ``str``
        :param filter_func function to determine whether a file shall be
            deleted or not.
        :type filter_func ``callable``

        """
        if os.path.exists(dirpath):
            for filename in os.listdir(dirpath):
                if not filter_func(filename):
                    continue
                path = os.path.join(dirpath, filename)
                if os.path.isdir(path):
                    shutil.rmtree(path)
                else:
                    os.unlink(path)
                self.logger.debug('deleted : %r', path) 

def do_iteration(self, mutation_fn, aggression):
        """
        This method is called with an output mutation filename and a
        real aggression (see :ref:`aggression`) indicating the amount
        of aggression the fuzzing algorithm should use.  *mutation_fn*
        is unique for every invokation of :meth:`do_iteration`.

        It is an error for this method not to write the mutated
        template to *mutation_fn* before returning a result.  If a
        result is not found, the mutation filename may be written to
        if needed, but it is not required.

        If a notable result is found, it should be returned as a
        :class:`FuzzResult` instance.  This will be stored and reported
        to the ALF central server at the next check-in interval.  A
        return of None indicates a result was not found.

        The filenames of any temporary files or folders created during
        execution can be safely removed using :func:`alf.delete`.  This
        is safer than using :func:`os.remove` or :func:`shutil.rmtree`
        directly.  *mutation_fn* does not need to be deleted, it is
        cleaned up automatically.
        """
        raise NotImplementedError() 

def test_creation(self):
        if os.path.exists(self.base_dir):
            shutil.rmtree(self.checkpoints_dir, ignore_errors=True)

        try:
            FileStructManager(base_dir=self.base_dir, is_continue=False)
        except FileStructManager.FSMException as err:
            self.fail("Raise error when base directory exists: [{}]".format(err))

        self.assertFalse(os.path.exists(self.base_dir))

        try:
            FileStructManager(base_dir=self.base_dir, is_continue=False)
        except FileStructManager.FSMException as err:
            self.fail("Raise error when base directory exists but empty: [{}]".format(err))

        os.makedirs(os.path.join(self.base_dir, 'new_dir'))
        try:
            FileStructManager(base_dir=self.base_dir, is_continue=False)
        except:
            self.fail("Error initialize when exists non-registered folders in base directory")

        shutil.rmtree(self.base_dir, ignore_errors=True) 

def __call__(self, args, env):
        samples_dir = env.get('samples-dir')
        for label in ("cat", "dog"):
            dest = os.path.join(samples_dir, label)
            if os.path.exists(dest):
                raise VergeMLError("Directory {} already exists in samples dir: {}".format(label, dest))
        print("Downloading cats and dogs to {}.".format(samples_dir))
        src_dir = self.download_files([(_URL, "catsdogs.zip")], env)
        path = os.path.join(src_dir, "catsdogs.zip")

        print("Extracting data.")
        zipf = zipfile.ZipFile(path, 'r')
        zipf.extractall(src_dir)
        zipf.close()

        for file, dest in (("PetImages/Dog", "dog"), ("PetImages/Cat", "cat")):
            shutil.copytree(os.path.join(src_dir, file), os.path.join(samples_dir, dest))

        shutil.rmtree(src_dir)

        # WTF?
        os.unlink(os.path.join(samples_dir, "cat", "666.jpg"))
        os.unlink(os.path.join(samples_dir, "dog", "11702.jpg"))

        print("Finished downloading cats and dogs.") 

def __call__(self, args, env):
        samples_dir = env.get('samples-dir')

        print("Downloading unique objects to {}.".format(samples_dir))

        src_dir = self.download_files([_URL], env=env, dir=env.get('cache-dir'))
        path = os.path.join(src_dir, "ObjectsAll.zip")

        zipf = zipfile.ZipFile(path, 'r')
        zipf.extractall(src_dir)
        zipf.close()

        for file in os.listdir(os.path.join(src_dir, "OBJECTSALL")):
            shutil.copy(os.path.join(src_dir, "OBJECTSALL", file), samples_dir)

        shutil.rmtree(src_dir)

        print("Finished downloading unique objects.") 

def _download_straight_dope_notebooks():
    """Downloads the Straight Dope Notebooks.

    Returns:
        True if it succeeds in downloading the notebooks without error.
    """
    logging.info('Cleaning and setting up notebooks directory "{}"'.format(NOTEBOOKS_DIR))
    shutil.rmtree(NOTEBOOKS_DIR, ignore_errors=True)

    cmd = [GIT_PATH,
           'clone',
           GIT_REPO,
           NOTEBOOKS_DIR]

    proc, msg = _run_command(cmd)

    if proc.returncode != 0:
        err_msg = 'Error downloading Straight Dope notebooks.\n'
        err_msg += msg
        logging.error(err_msg)
        return False
    return True 

def teardownClass(cls):
        if cls.IMAGES_DIR:
            print("cleanup {}".format(cls.IMAGES_DIR))
            shutil.rmtree(cls.IMAGES_DIR) 

def remove_directory_temp(self):
		"""
		Delete temporary data

		@return: Nothing
		@rtype: None
		"""
		if os.path.exists(self._tmp_dir):
			assert os.path.isdir(self._tmp_dir)
			shutil.rmtree(self._tmp_dir) 

def tearDown(self):
        # Remove the directory after the test
        shutil.rmtree(self.test_dir) 

def test_metrics_processing(self):
        def check_data():
            self.assertIn('d', data)
            self.assertIn('lv1', data)
            self.assertIn('lv2', data['lv1'])
            self.assertIn('a', data['lv1'])
            self.assertIn('b', data['lv1'])
            self.assertIn('c', data['lv1']['lv2'])

            self.assertAlmostEqual(data['d'], values[-1] * 4, delta=1e-5)
            self.assertAlmostEqual(data['lv1']['a'], values[-1], delta=1e-5)
            self.assertAlmostEqual(data['lv1']['b'], values[-1] * 2, delta=1e-5)
            self.assertAlmostEqual(data['lv1']['lv2']['c'], values[-1] * 3, delta=1e-5)

        self.metrics_processing(with_final_file=False)

        shutil.rmtree('data')

        values = self.metrics_processing(with_final_file=True)
        expected_out = os.path.join('data', 'monitors', 'metrics_log', 'metrics.json')
        self.assertTrue(os.path.exists(expected_out) and os.path.isfile(expected_out))

        with open(expected_out, 'r') as file:
            data = json.load(file)

        check_data()

        shutil.rmtree('data')

        values = self.metrics_processing(with_final_file=True, final_file='my_metrics.json')
        expected_out = os.path.join('my_metrics.json')
        self.assertTrue(os.path.exists(expected_out) and os.path.isfile(expected_out))

        with open(expected_out, 'r') as file:
            data = json.load(file)

        check_data() 

def tearDownClass(cls):
        os.chdir(CWD)
        shutil.rmtree(cls.test_dir) 

def tmpdir(prefix='npythy_tempdir_', delete=True):
    '''
    tmpdir() creates a temporary directory and yields its path. At python exit, the directory and
      all of its contents are recursively deleted (so long as the the normal python exit process is
      allowed to call the atexit handlers).
    tmpdir(prefix) uses the given prefix in the tempfile.mkdtemp() call.
    
    The option delete may be set to False to specify that the tempdir should not be deleted on exit.
    '''
    path = tempfile.mkdtemp(prefix=prefix)
    if not os.path.isdir(path): raise ValueError('Could not find or create temp directory')
    if delete: atexit.register(shutil.rmtree, path)
    return path 

def load_data(config):
    """
    :param config:  config
    :return: batch data iterator and alphabet
    """
    print("load data for process or pkl data.")
    train_iter, dev_iter, test_iter = None, None, None
    alphabet = None
    start_time = time.time()
    if (config.train is True) and (config.process is True):
        print("process data")
        if os.path.exists(config.pkl_directory): shutil.rmtree(config.pkl_directory)
        if not os.path.isdir(config.pkl_directory): os.makedirs(config.pkl_directory)
        train_iter, dev_iter, test_iter, alphabet = preprocessing(config)
        config.pretrained_weight = pre_embed(config=config, alphabet=alphabet)
    elif ((config.train is True) and (config.process is False)) or (config.test is True):
        print("load data from pkl file")
        # load alphabet from pkl
        # alphabet_dict = pcl.load(path=os.path.join(config.pkl_directory, config.pkl_alphabet))
        alphabet_dict = torch.load(f=os.path.join(config.pkl_directory, config.pkl_alphabet))
        print(alphabet_dict.keys())
        alphabet = alphabet_dict["alphabet"]
        # load iter from pkl
        # iter_dict = pcl.load(path=os.path.join(config.pkl_directory, config.pkl_iter))
        iter_dict = torch.load(f=os.path.join(config.pkl_directory, config.pkl_iter))
        print(iter_dict.keys())
        train_iter, dev_iter, test_iter = iter_dict.values()
        # train_iter, dev_iter, test_iter = iter_dict["train_iter"], iter_dict["dev_iter"], iter_dict["test_iter"]
        # load embed from pkl
        config.pretrained_weight = None
        if os.path.exists(os.path.join(config.pkl_directory, config.pkl_embed)):
            # embed_dict = pcl.load(os.path.join(config.pkl_directory, config.pkl_embed))
            embed_dict = torch.load(f=os.path.join(config.pkl_directory, config.pkl_embed))
            print(embed_dict.keys())
            embed = embed_dict["pretrain_embed"]
            config.pretrained_weight = embed
    end_time = time.time()
    print("All Data/Alphabet/Iterator Use Time {:.4f}".format(end_time - start_time))
    print("***************************************")
    return train_iter, dev_iter, test_iter, alphabet 

def setup_cache_deletion(cache_path, actual_cache_path, cache_delete):
        '''
        The filemap.setup_cache_deletion requirement ensures that the cache directory will be
        deleted at system exit, if required.
        '''
        if cache_delete is True and cache_path is actual_cache_path:
            atexit.register(shutil.rmtree, cache_path)
        return True 

def cache_root(custom_directory):
        '''
        dataset.cache_root is the root directory in which the given dataset has been cached.
        '''
        if custom_directory is not None: return None
        elif config['data_cache_root'] is None:
            # we create a data-cache in a temporary directory
            path = tempfile.mkdtemp(prefix='npythy_data_cache_')
            if not os.path.isdir(path): raise ValueError('Could not find or create cache directory')
            config['data_cache_root'] = path
            atexit.register(shutil.rmtree, path)
        return config['data_cache_root'] 

def test_download_successful():
    """ test download with one process """
    tmp = tempfile.mkdtemp()
    tmpfile = os.path.join(tmp, 'README.md')
    _download_successful(tmpfile)
    assert os.path.getsize(tmpfile) > 100, os.path.getsize(tmpfile)
    pattern = os.path.join(tmp, 'README.md*')
    # check only one file we want left
    assert len(glob.glob(pattern)) == 1, glob.glob(pattern)
    # delete temp dir
    shutil.rmtree(tmp) 

def get_params(config, alphabet):
    """
    :param config: config
    :param alphabet:  alphabet dict
    :return:
    """
    # get algorithm
    config.learning_algorithm = get_learning_algorithm(config)

    # save best model path
    config.save_best_model_path = config.save_best_model_dir
    if config.test is False:
        if os.path.exists(config.save_best_model_path):
            shutil.rmtree(config.save_best_model_path)

    # get params
    config.embed_num = alphabet.word_alphabet.vocab_size
    config.char_embed_num = alphabet.char_alphabet.vocab_size
    config.class_num = alphabet.label_alphabet.vocab_size
    config.paddingId = alphabet.word_paddingId
    config.char_paddingId = alphabet.char_paddingId
    config.label_paddingId = alphabet.label_paddingId
    config.create_alphabet = alphabet
    print("embed_num : {}, char_embed_num: {}, class_num : {}".format(config.embed_num, config.char_embed_num,
                                                                      config.class_num))
    print("PaddingID {}".format(config.paddingId))
    print("char PaddingID {}".format(config.char_paddingId)) 

def _close(self):
        if self.validate_dir(self._temp_merges_bam_directory, silent=True):
            shutil.rmtree(self._temp_merges_bam_directory)
        self._logger = None 

def tearDownClass(cls):
        os.chdir(CWD)
        shutil.rmtree(cls.test_dir) 

def __del__(self):
        super(NcbiTaxonomy, self).__del__()
        if self.validate_dir(self._tmp_dir, silent=True):
            import shutil
            shutil.rmtree(self._tmp_dir)
        self.tmp_dir = None 

def __exit__(self, type, value, traceback):
        super(NcbiTaxonomy, self).__exit__(type, value, traceback)
        if self.validate_dir(self._tmp_dir, silent=True):
            import shutil
            shutil.rmtree(self._tmp_dir)
        self.tmp_dir = None 

def __exit__(self, type, value, traceback):
		super(ANIm, self).__exit__(type, value, traceback)
		if not self._debug:
			shutil.rmtree(self._tmp_dir)

	# def get_total_organism_length(self):
	# 	self.get_organism_lengths(self._reference_gid_to_location)
	# 	self.get_organism_lengths(self._query_gid_to_location)

	# Construct a command-line for NUCmer reference_id, candidate_id 

def __exit__(self, type, value, traceback):
		super(MGCluster, self).__exit__(type, value, traceback)
		if not self._debug:
			shutil.rmtree(self._tmp_dir) 

def inflate_data_sources(input):
    import zipfile
    import tempfile
    import shutil
    import os
    dirpath = tempfile.mkdtemp()
    # make sure the directory is empty
    shutil.rmtree(dirpath)
    os.makedirs(dirpath)
    # load and extract zip file
    zf = zipfile.ZipFile(input)
    zf.extractall(dirpath)
    extracted_folders = os.listdir(dirpath)
    return {k.split(".")[0]: os.path.join(dirpath, k) for k in extracted_folders} 

def _remove(path):
        """ param <path> could either be relative or absolute. """
        if os.path.isfile(path):
            os.remove(path)  # remove the file
        elif os.path.isdir(path):
            import shutil
            shutil.rmtree(path)  # remove dir and all contains 

def _remove(path):
        """ param <path> could either be relative or absolute. """
        if os.path.isfile(path):
            os.remove(path)  # remove the file
        elif os.path.isdir(path):
            import shutil
            shutil.rmtree(path)  # remove dir and all contains 

def test_label_dir(self):
        # prepare the images
        p = env.get_data_folder()
        api = ImagenetAPI(p, limit=3)
        api.gather("n01316949", include_subset=True)

        folder = "work_animal"

        # make labeled data
        machine = LabelingMachine(p)

        lf_fixed = machine.label_dir(0, path_from_root=folder, label_file=os.path.join(p, "test_label_fixed.txt"))
        lf_auto, label_def = machine.label_dir_auto(path_from_root=folder, label_file=os.path.join(p, "test_label_auto.txt"))

        f_count = 0
        for im in lf_fixed.fetch():
            self.assertEqual(0, im.label)
            f_count += 1

        label_and_path = {}
        a_count = 0
        with open(label_def) as f:
            path_label = [ln.strip().split() for ln in f.readlines()]
            for pl in path_label:
                label_and_path[int(pl[0])] = pl[1]

        for im in lf_auto.fetch():
            path = os.path.dirname(im.path)
            rel_path = machine.file_api.to_rel(path)
            self.assertTrue(im.label in label_and_path)
            self.assertEqual(os.path.join(label_and_path[im.label], ""), os.path.join(rel_path, ""))
            a_count += 1
        else:
            im = None # release reference

        self.assertGreater(f_count, 0)
        self.assertEquals(f_count, a_count)
        shutil.rmtree(api.file_api.to_abs(folder))
        os.remove(lf_fixed.path)
        os.remove(lf_auto.path)
        os.remove(label_def)