Python shutil.move() Examples

def test_move_dir_caseinsensitive(self):
        # Renames a folder to the same name
        # but a different case.

        self.src_dir = tempfile.mkdtemp()
        dst_dir = os.path.join(
                os.path.dirname(self.src_dir),
                os.path.basename(self.src_dir).upper())
        self.assertNotEqual(self.src_dir, dst_dir)

        try:
            shutil.move(self.src_dir, dst_dir)
            self.assertTrue(os.path.isdir(dst_dir))
        finally:
            if os.path.exists(dst_dir):
                os.rmdir(dst_dir) 

def _do_copyish(self, filename1, filename2):
        # Should be able to rename the file using either name.
        self.assertTrue(os.path.isfile(filename1)) # must exist.
        os.rename(filename1, filename2 + ".new")
        self.assertTrue(os.path.isfile(filename1+".new"))
        os.rename(filename1 + ".new", filename2)
        self.assertTrue(os.path.isfile(filename2))

        shutil.copy(filename1, filename2 + ".new")
        os.unlink(filename1 + ".new") # remove using equiv name.
        # And a couple of moves, one using each name.
        shutil.move(filename1, filename2 + ".new")
        self.assertTrue(not os.path.exists(filename2))
        shutil.move(filename1 + ".new", filename2)
        self.assertTrue(os.path.exists(filename1))
        # Note - due to the implementation of shutil.move,
        # it tries a rename first.  This only fails on Windows when on
        # different file systems - and this test can't ensure that.
        # So we test the shutil.copy2 function, which is the thing most
        # likely to fail.
        shutil.copy2(filename1, filename2 + ".new")
        os.unlink(filename1 + ".new") 

def restore_config_file():
    os.remove(zmirror_file('config.py'))
    os.remove(zmirror_file('custom_func.py'))
    if os.path.exists(zmirror_file('config.py._unittest_raw')):
        shutil.move(zmirror_file('config.py._unittest_raw'), zmirror_file('config.py'))
    if os.path.exists(zmirror_file('custom_func.py._unittest_raw')):
        shutil.move(zmirror_file('custom_func.py._unittest_raw'), zmirror_file('custom_func.py'))

    try:
        os.remove(zmirror_file('ip_whitelist.txt'))
    except:
        pass
    try:
        os.remove(zmirror_file('ip_whitelist.log'))
    except:
        pass
    try:
        os.remove(zmirror_file('automatic_domains_whitelist.log'))
    except:
        pass 

def extract_tar(complete_folder, tar_file, data_name):
    """
    Extract tar.gz file and get the data.
    :param tar_file:
    :return:
    """
    files_folder = os.listdir(complete_folder)
    tar_file_path = os.path.join(complete_folder, tar_file)
    bash_command = ['tar', 'xzf', tar_file_path,'-C', complete_folder]
    subprocess.run(bash_command)
    new_files_folder = os.listdir(complete_folder)
    for f in new_files_folder:
        if f not in files_folder:  # Extracted file or folder
            if data_name == f:  # It is the file we were expecting
                break
            elif os.path.isdir(f):  # It is a folder
                subfolder_path = os.path.join(complete_folder, f)
                files_in_folder = os.listdir(subfolder_path)
                for f_s in files_in_folder:
                    if data_name == f_s:
                        shutil.move(os.path.join(subfolder_path, data_name),
                                    os.path.join(complete_folder, data_name))
                        shutil.rmtree(subfolder_path)
                        os.remove(tar_file_path)
                        break 

def _build_one_inside_env(self, req, output_dir, python_tag=None):
        with TempDirectory(kind="wheel") as temp_dir:
            if self.__build_one(req, temp_dir.path, python_tag=python_tag):
                try:
                    wheel_name = os.listdir(temp_dir.path)[0]
                    wheel_path = os.path.join(output_dir, wheel_name)
                    shutil.move(
                        os.path.join(temp_dir.path, wheel_name), wheel_path
                    )
                    logger.info('Stored in directory: %s', output_dir)
                    return wheel_path
                except:
                    pass
            # Ignore return, we can't do anything else useful.
            self._clean_one(req)
            return None 

def _copy_file(filename, location, link):
    copy = True
    download_location = os.path.join(location, link.filename)
    if os.path.exists(download_location):
        response = ask_path_exists(
            'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)abort' %
            display_path(download_location), ('i', 'w', 'b', 'a'))
        if response == 'i':
            copy = False
        elif response == 'w':
            logger.warning('Deleting %s', display_path(download_location))
            os.remove(download_location)
        elif response == 'b':
            dest_file = backup_dir(download_location)
            logger.warning(
                'Backing up %s to %s',
                display_path(download_location),
                display_path(dest_file),
            )
            shutil.move(download_location, dest_file)
        elif response == 'a':
            sys.exit(-1)
    if copy:
        shutil.copy(filename, download_location)
        logger.info('Saved %s', display_path(download_location)) 

def _copy_file(filename, location, content_type, link):
    copy = True
    download_location = os.path.join(location, link.filename)
    if os.path.exists(download_location):
        response = ask_path_exists(
            'The file %s exists. (i)gnore, (w)ipe, (b)ackup ' %
            display_path(download_location), ('i', 'w', 'b'))
        if response == 'i':
            copy = False
        elif response == 'w':
            logger.warn('Deleting %s' % display_path(download_location))
            os.remove(download_location)
        elif response == 'b':
            dest_file = backup_dir(download_location)
            logger.warn('Backing up %s to %s'
                        % (display_path(download_location), display_path(dest_file)))
            shutil.move(download_location, dest_file)
    if copy:
        shutil.copy(filename, download_location)
        logger.notify('Saved %s' % display_path(download_location)) 

def _compile_module_file(template, text, filename, outputpath, module_writer):
    source, lexer = _compile(template, text, filename,
                        generate_magic_comment=True)

    if isinstance(source, compat.text_type):
        source = source.encode(lexer.encoding or 'ascii')

    if module_writer:
        module_writer(source, outputpath)
    else:
        # make tempfiles in the same location as the ultimate
        # location.   this ensures they're on the same filesystem,
        # avoiding synchronization issues.
        (dest, name) = tempfile.mkstemp(dir=os.path.dirname(outputpath))

        os.write(dest, source)
        os.close(dest)
        shutil.move(name, outputpath) 

def maybe_move(self, spec, dist_filename, setup_base):
        dst = os.path.join(self.build_directory, spec.key)
        if os.path.exists(dst):
            msg = "%r already exists in %s; build directory %s will not be kept"
            log.warn(msg, spec.key, self.build_directory, setup_base)
            return setup_base
        if os.path.isdir(dist_filename):
            setup_base = dist_filename
        else:
            if os.path.dirname(dist_filename)==setup_base:
                os.unlink(dist_filename)   # get it out of the tmp dir
            contents = os.listdir(setup_base)
            if len(contents)==1:
                dist_filename = os.path.join(setup_base,contents[0])
                if os.path.isdir(dist_filename):
                    # if the only thing there is a directory, move it instead
                    setup_base = dist_filename
        ensure_directory(dst)
        shutil.move(setup_base, dst)
        return dst 

def _process_mask(self, layer_dir, mask_filename):
        """Process an incoming mask file

        Args:
            layer_dir : The directory to store the mask sym link.
            mask_filename : The original mask that will be converted to MXE 
                format and linked in the layer directory.
        """
        work_mask_filename = os.path.join(layer_dir,
                                          'mask{}'.format(LMFormat.MXE.ext))
        if not os.path.exists(work_mask_filename):
            convertAsciisToMxes(os.path.split(mask_filename)[0])
            shutil.move('{}{}'.format(
                os.path.splitext(os.path.abspath(mask_filename))[0], 
                LMFormat.MXE.ext), work_mask_filename)
        return 'togglelayertype=mask'
        
    # ................................... 

def setup_logging(log_loc):
    if os.path.exists(log_loc):
        shutil.move(log_loc, log_loc + "_" + str(int(os.path.getctime(log_loc))))
    os.makedirs(log_loc)

    log_file = '{}/benchmark.log'.format(log_loc)
    LOGGER = logging.getLogger('benchmark')
    LOGGER.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s %(levelname)s:%(name)s %(message)s')
    file_handler = logging.FileHandler(log_file)
    console_handler = logging.StreamHandler()
    file_handler.setFormatter(formatter)
    console_handler.setFormatter(formatter)

    LOGGER.addHandler(file_handler)
    LOGGER.addHandler(console_handler)
    return LOGGER 

def download_aug(path, overwrite=False):
    _AUG_DOWNLOAD_URLS = [
        ('http://www.eecs.berkeley.edu/Research/Projects/CS/vision/grouping/semantic_contours/benchmark.tgz', '7129e0a480c2d6afb02b517bb18ac54283bfaa35')]
    makedirs(path)
    for url, checksum in _AUG_DOWNLOAD_URLS:
        filename = download(url, path=path, overwrite=overwrite, sha1_hash=checksum)
        # extract
        with tarfile.open(filename) as tar:
            tar.extractall(path=path)
            shutil.move(os.path.join(path, 'benchmark_RELEASE'),
                        os.path.join(path, 'VOCaug'))
            filenames = ['VOCaug/dataset/train.txt', 'VOCaug/dataset/val.txt']
            # generate trainval.txt
            with open(os.path.join(path, 'VOCaug/dataset/trainval.txt'), 'w') as outfile:
                for fname in filenames:
                    fname = os.path.join(path, fname)
                    with open(fname) as infile:
                        for line in infile:
                            outfile.write(line) 

def cb(req):
    global evp
    if req.start_recording:
        if evp is not None and evp.started:
            evp.stop()
        evp = ExperimentVideoProc()
        evp.start()
    else:
        if evp is None or not evp.started:
            pass
        else:
            evp.stop() 
            if req.experiment_name == "":
                req.experiment_name = "None"
            experiment_mp4_path = os.path.join(experiment_video_folder, "%s.mp4"%req.experiment_name)
            shutil.move(
                default_mp4_path,
                experiment_mp4_path 
            )

    return ExperimentRecordingResponse(True) 

def delete_entry(self, name, domain):
        if name is None:
            raise exception.InvalidInput(_("Invalid name"))

        deleted = False
        outfile = tempfile.NamedTemporaryFile('w', delete=False)
        with open(self.filename, 'r') as infile:
            for line in infile:
                entry = self.parse_line(line)
                if (not entry or
                        entry['name'] != self.qualify(name, domain)):
                    outfile.write(line)
                else:
                    deleted = True
        outfile.close()
        shutil.move(outfile.name, self.filename)
        if not deleted:
            LOG.warning(_LW('Cannot delete entry |%s|'),
                        self.qualify(name, domain))
            raise exception.NotFound 

def modify_address(self, name, address, domain):

        if not self.get_entries_by_name(name, domain):
            raise exception.NotFound

        outfile = tempfile.NamedTemporaryFile('w', delete=False)
        with open(self.filename, 'r') as infile:
            for line in infile:
                entry = self.parse_line(line)
                if (entry and
                        entry['name'] == self.qualify(name, domain)):
                    outfile.write("%s   %s   %s\n" %
                        (address, self.qualify(name, domain), entry['type']))
                else:
                    outfile.write(line)
        outfile.close()
        shutil.move(outfile.name, self.filename) 

def delete_domain(self, fqdomain):
        deleted = False
        outfile = tempfile.NamedTemporaryFile('w', delete=False)
        with open(self.filename, 'r') as infile:
            for line in infile:
                entry = self.parse_line(line)
                if (not entry or
                        entry['domain'] != fqdomain.lower()):
                    outfile.write(line)
                else:
                    LOG.info(_LI("deleted %s"), entry)
                    deleted = True
        outfile.close()
        shutil.move(outfile.name, self.filename)
        if not deleted:
            LOG.warning(_LW('Cannot delete domain |%s|'), fqdomain)
            raise exception.NotFound 

def _build_one(self, req, output_dir, python_tag=None):
        """Build one wheel.

        :return: The filename of the built wheel, or None if the build failed.
        """
        tempd = tempfile.mkdtemp('pip-wheel-')
        try:
            if self.__build_one(req, tempd, python_tag=python_tag):
                try:
                    wheel_name = os.listdir(tempd)[0]
                    wheel_path = os.path.join(output_dir, wheel_name)
                    shutil.move(os.path.join(tempd, wheel_name), wheel_path)
                    logger.info('Stored in directory: %s', output_dir)
                    return wheel_path
                except:
                    pass
            # Ignore return, we can't do anything else useful.
            self._clean_one(req)
            return None
        finally:
            rmtree(tempd) 

def _copy_file(filename, location, link):
    copy = True
    download_location = os.path.join(location, link.filename)
    if os.path.exists(download_location):
        response = ask_path_exists(
            'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)abort' %
            display_path(download_location), ('i', 'w', 'b', 'a'))
        if response == 'i':
            copy = False
        elif response == 'w':
            logger.warning('Deleting %s', display_path(download_location))
            os.remove(download_location)
        elif response == 'b':
            dest_file = backup_dir(download_location)
            logger.warning(
                'Backing up %s to %s',
                display_path(download_location),
                display_path(dest_file),
            )
            shutil.move(download_location, dest_file)
        elif response == 'a':
            sys.exit(-1)
    if copy:
        shutil.copy(filename, download_location)
        logger.info('Saved %s', display_path(download_location)) 

def _replace_zip_directory_cache_data(normalized_path):
        def replace_cached_zip_archive_directory_data(path, old_entry):
            # N.B. In theory, we could load the zip directory information just
            # once for all updated path spellings, and then copy it locally and
            # update its contained path strings to contain the correct
            # spelling, but that seems like a way too invasive move (this cache
            # structure is not officially documented anywhere and could in
            # theory change with new Python releases) for no significant
            # benefit.
            old_entry.clear()
            zipimport.zipimporter(path)
            old_entry.update(zipimport._zip_directory_cache[path])
            return old_entry

        _update_zipimporter_cache(
            normalized_path, zipimport._zip_directory_cache,
            updater=replace_cached_zip_archive_directory_data) 

def maybe_move(self, spec, dist_filename, setup_base):
        dst = os.path.join(self.build_directory, spec.key)
        if os.path.exists(dst):
            msg = (
                "%r already exists in %s; build directory %s will not be kept"
            )
            log.warn(msg, spec.key, self.build_directory, setup_base)
            return setup_base
        if os.path.isdir(dist_filename):
            setup_base = dist_filename
        else:
            if os.path.dirname(dist_filename) == setup_base:
                os.unlink(dist_filename)  # get it out of the tmp dir
            contents = os.listdir(setup_base)
            if len(contents) == 1:
                dist_filename = os.path.join(setup_base, contents[0])
                if os.path.isdir(dist_filename):
                    # if the only thing there is a directory, move it instead
                    setup_base = dist_filename
        ensure_directory(dst)
        shutil.move(setup_base, dst)
        return dst 

def _replace_zip_directory_cache_data(normalized_path):
        def replace_cached_zip_archive_directory_data(path, old_entry):
            # N.B. In theory, we could load the zip directory information just
            # once for all updated path spellings, and then copy it locally and
            # update its contained path strings to contain the correct
            # spelling, but that seems like a way too invasive move (this cache
            # structure is not officially documented anywhere and could in
            # theory change with new Python releases) for no significant
            # benefit.
            old_entry.clear()
            zipimport.zipimporter(path)
            old_entry.update(zipimport._zip_directory_cache[path])
            return old_entry

        _update_zipimporter_cache(
            normalized_path, zipimport._zip_directory_cache,
            updater=replace_cached_zip_archive_directory_data) 

def get_return_label(self):
        """Get the GSX return label for this part."""
        if self.return_label.name == "":
            # Return label not yet set, get it...
            label = gsxws.Return(self.return_order).get_label(self.part_number)
            filename = "%s_%s.pdf" % (self.return_order, self.part_number)
            tmp_fp = os.path.join(settings.TEMP_ROOT, filename)
            #  move the label to local temp to avoid
            #  django.security.SuspiciousFileOperation
            shutil.move(label.returnLabelFileData, tmp_fp)

            f = File(open(tmp_fp, 'r'))
            self.return_label = f
            self.save()
            self.return_label.save(filename, f)
            f.closed
            os.remove(tmp_fp)

        return self.return_label.read() 

def _build_one(self, req, output_dir, python_tag=None):
        """Build one wheel.

        :return: The filename of the built wheel, or None if the build failed.
        """
        tempd = tempfile.mkdtemp('pip-wheel-')
        try:
            if self.__build_one(req, tempd, python_tag=python_tag):
                try:
                    wheel_name = os.listdir(tempd)[0]
                    wheel_path = os.path.join(output_dir, wheel_name)
                    shutil.move(os.path.join(tempd, wheel_name), wheel_path)
                    logger.info('Stored in directory: %s', output_dir)
                    return wheel_path
                except:
                    pass
            # Ignore return, we can't do anything else useful.
            self._clean_one(req)
            return None
        finally:
            rmtree(tempd) 

def _copy_file(filename, location, link):
    copy = True
    download_location = os.path.join(location, link.filename)
    if os.path.exists(download_location):
        response = ask_path_exists(
            'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)abort' %
            display_path(download_location), ('i', 'w', 'b', 'a'))
        if response == 'i':
            copy = False
        elif response == 'w':
            logger.warning('Deleting %s', display_path(download_location))
            os.remove(download_location)
        elif response == 'b':
            dest_file = backup_dir(download_location)
            logger.warning(
                'Backing up %s to %s',
                display_path(download_location),
                display_path(dest_file),
            )
            shutil.move(download_location, dest_file)
        elif response == 'a':
            sys.exit(-1)
    if copy:
        shutil.copy(filename, download_location)
        logger.info('Saved %s', display_path(download_location)) 

def _replace_zip_directory_cache_data(normalized_path):
        def replace_cached_zip_archive_directory_data(path, old_entry):
            # N.B. In theory, we could load the zip directory information just
            # once for all updated path spellings, and then copy it locally and
            # update its contained path strings to contain the correct
            # spelling, but that seems like a way too invasive move (this cache
            # structure is not officially documented anywhere and could in
            # theory change with new Python releases) for no significant
            # benefit.
            old_entry.clear()
            zipimport.zipimporter(path)
            old_entry.update(zipimport._zip_directory_cache[path])
            return old_entry

        _update_zipimporter_cache(
            normalized_path, zipimport._zip_directory_cache,
            updater=replace_cached_zip_archive_directory_data) 

def compress_local_copyto(self, path):
        # this function compresses and moves the local copy of the rinex
        # meant to be used when a multiday rinex file is encountered and we need to move it to the repository

        # compress the rinex into crinez. Make the filename
        crinez = self.to_format(self.rinex, TYPE_CRINEZ)

        # we make the crinez again (don't use the existing from the database) to apply any corrections
        # made during the __init__ stage. Notice the -f in rnx2crz
        cmd = pyRunWithRetry.RunCommand('rnx2crz -f ' + self.rinex_path, 45)
        try:
            _, err = cmd.run_shell()

            if os.path.getsize(os.path.join(self.rootdir, crinez)) == 0:
                raise pyRinexException('Error in compress_local_copyto: compressed version of ' + self.rinex_path + ' has zero size!')
        except pyRunWithRetry.RunCommandWithRetryExeception as e:
            # catch the timeout except and pass it as a pyRinexException
            raise pyRinexException(str(e))

        filename = Utils.copyfile(os.path.join(self.rootdir, crinez), os.path.join(path, crinez))

        self.log_event('Created CRINEZ from local copy and copied to %s' % path)

        return filename 

def fix_plugin_directories(dest):
    """
    Loops through all folders in the output directory, reads the their manifest
    file, and renames the directory to the standard <platform>_<guid> format
    """
    # Loop through directories in the destination directory
    for existing_dir in os.listdir(dest):
        existing_path = os.path.join(dest, existing_dir)

        # Skip non-directories
        if not os.path.isdir(existing_path):
            continue

        try:
            with open(os.path.join(existing_path, 'manifest.json')) as m:
                data = json.load(m)
                platform = data['platform']
                guid = data['guid']

                # Close json file
                m.close()

                expected_dir = platform + '_' + guid
                expected_path = os.path.join(dest, expected_dir)

                if existing_path != expected_path:
                    print('NOTICE: Folder should be "{}", but it is named "{}"'
                          .format(expected_dir, existing_dir))

                    if os.path.isdir(expected_path):
                        print('NOTICE: Correct pathed plugin already exists,'
                              + ' deleting extra plugin')
                        shutil.rmtree(existing_path)
                    else:
                        print('NOTICE: Renaming folder to proper name')
                        shutil.move(existing_path, expected_path)
        except (FileNotFoundError, json.decoder.JSONDecodeError, KeyError):
            print('ERROR: Could not read plugin data from {} folder'
                  .format(existing_path)) 

def _check_move_file(self, src, dst, real_dst):
        with open(src, "rb") as f:
            contents = f.read()
        shutil.move(src, dst)
        with open(real_dst, "rb") as f:
            self.assertEqual(contents, f.read())
        self.assertFalse(os.path.exists(src)) 

def _check_move_dir(self, src, dst, real_dst):
        contents = sorted(os.listdir(src))
        shutil.move(src, dst)
        self.assertEqual(contents, sorted(os.listdir(real_dst)))
        self.assertFalse(os.path.exists(src)) 

def test_existing_file_inside_dest_dir(self):
        # A file with the same name inside the destination dir already exists.
        with open(self.dst_file, "wb"):
            pass
        self.assertRaises(shutil.Error, shutil.move, self.src_file, self.dst_dir) 

def test_dont_move_dir_in_itself(self):
        # Moving a dir inside itself raises an Error.
        dst = os.path.join(self.src_dir, "bar")
        self.assertRaises(shutil.Error, shutil.move, self.src_dir, dst) 

def create_epub(extracted_epub_path, original_epub_path):
    original_epub_basename = os.path.split(original_epub_path)[1]
    original_epub_dir = os.path.split(original_epub_path)[0]
    # print(original_epub_dir)
    # print(original_epub_basename)
    new_epub_name = os.path.splitext(original_epub_basename)[
        0] + "_highlighted.epub"
    # print(new_epub_name)
    # print(extracted_epub_path)
    new_epub_path = original_epub_dir + "/" + new_epub_name
    # print(new_epub_path)
    zip_path = distutils.archive_util.make_archive(
        new_epub_name, format='zip', root_dir=extracted_epub_path)
    shutil.move(zip_path, new_epub_path + '.zip')
    os.rename(new_epub_path + '.zip', new_epub_path.replace('zip', '')) 

def complete(self):
        assert self.status is not None
        files = glob.glob(os.path.join(RUNNING_PATH, '%d.*' % self.id))
        for x in files:
            shutil.move(x, os.path.join(COMPLETED_PATH, os.path.basename(x))) 

def download(url, out=None, bar=bar_adaptive):
    """High level function, which downloads URL into tmp file in current
    directory and then renames it to filename autodetected from either URL
    or HTTP headers.

    :param bar: function to track download progress (visualize etc.)
    :param out: output filename or directory
    :return:    filename where URL is downloaded to
    """
    names = dict()
    names["out"] = out or ''
    names["url"] = filename_from_url(url)
    # get filename for temp file in current directory
    prefix = (names["url"] or names["out"] or ".") + "."
    (fd, tmpfile) = tempfile.mkstemp(".tmp", prefix=prefix, dir=".")
    os.close(fd)
    os.unlink(tmpfile)

    # set progress monitoring callback
    def callback_charged(blocks, block_size, total_size):
        # 'closure' to set bar drawing function in callback
        callback_progress(blocks, block_size, total_size, bar_function=bar)
    if bar:
        callback = callback_charged
    else:
        callback = None

    (tmpfile, headers) = urllib.urlretrieve(url, tmpfile, callback)
    names["header"] = filename_from_headers(headers)
    if os.path.isdir(names["out"]):
        filename = names["header"] or names["url"]
        filename = names["out"] + "/" + filename
    else:
        filename = names["out"] or names["header"] or names["url"]
    # add numeric ' (x)' suffix if filename already exists
    if os.path.exists(filename):
        filename = filename_fix_existing(filename)
    shutil.move(tmpfile, filename)

    #print headers
    return filename 

def download_apkpure(driver, num):
    try:
        gp_file = "GooglePlayRank_{num}.txt".format(num=num)
        gp_file_tmp = "GooglePlayRankTmp_{num}.txt".format(num=num)
        with open(gp_file) as f_in:
            pkg = f_in.readline().replace('\n', '').strip()
        url = "https://m.apkpure.com/cn/search?q={pkg}"
        _url = url.format(pkg=pkg)
        # _url = "https://m.apkpure.com/cn/search?q=com.whatsapp"
        driver.maximize_window()
        driver.get(_url)
        driver.find_element_by_class_name("dd").click()
        driver.find_element_by_class_name("da").click()
        down_url = driver.find_element_by_id("download_link").get_attribute("href")
        print(down_url)
        print("download start")
        time.sleep(10)
        download_dir = os.path.join(os.getcwd(), "apk")
        while True:
            if check_file(download_dir):
                break
            print("downloading")
            time.sleep(5)
            continue
        print("download success")
        download_success_file = "GooglePlayRank_success_{num}.txt".format(num=num)
        with open(download_success_file, 'a+', encoding='utf-8') as f:
            _pkg = pkg + "\n"
            f.write(_pkg)
        return
    except Exception as e:
        print("download:", e)
    finally:
        with open(gp_file) as f_in:
            with open(gp_file_tmp, 'w') as f_tmp:
                for line in f_in.readlines():
                    if pkg not in line:
                        f_tmp.write(line)
        shutil.move(gp_file_tmp, gp_file)
        print("download:", pkg) 

def copy_default_config_file():
    if os.path.exists(zmirror_file('config.py')):
        print('[Waring] the config.py already exists, it would be temporary renamed to config.py._unittest_raw')
        shutil.move(zmirror_file('config.py'), zmirror_file('config.py._unittest_raw'))

    if os.path.exists(zmirror_file('custom_func.py')):
        print('[Waring] the custom_func.py already exists, it would be temporary renamed to custom_func.py._unittest_raw')
        shutil.move(zmirror_file('custom_func.py'), zmirror_file('custom_func.py._unittest_raw'))

    shutil.copy(zmirror_file('config_default.py'), zmirror_file('config.py'))
    shutil.copy(zmirror_file('custom_func.sample.py'), zmirror_file('custom_func.py'))

    try:
        os.remove(zmirror_file('ip_whitelist.txt'))
    except:
        pass
    try:
        os.remove(zmirror_file('ip_whitelist.log'))
    except:
        pass
    try:
        os.remove(zmirror_file('automatic_domains_whitelist.log'))
    except:
        pass

        # 下面是flask的一个trick, 强行生成多个不同的flask client 对象
        # with open(zmirror_file('config.py'), 'a', encoding='utf-8') as fp:
        #     fp.write('\n')
        #     fp.write('import random\n')
        #     fp.write('from flask import Flask\n')
        #     fp.write("unittest_app = Flask('unittest' + str(random.random()).replace('.', ''))\n") 

def _compile_module(self):
        # compile this C source
        tmpdir = os.path.dirname(self.sourcefilename)
        outputfilename = ffiplatform.compile(tmpdir, self.get_extension())
        try:
            same = ffiplatform.samefile(outputfilename, self.modulefilename)
        except OSError:
            same = False
        if not same:
            _ensure_dir(self.modulefilename)
            shutil.move(outputfilename, self.modulefilename)
        self._has_module = True 

def renames(old, new):
    """Like os.renames(), but handles renaming across devices."""
    # Implementation borrowed from os.renames().
    head, tail = os.path.split(new)
    if head and tail and not os.path.exists(head):
        os.makedirs(head)

    shutil.move(old, new)

    head, tail = os.path.split(old)
    if head and tail:
        try:
            os.removedirs(head)
        except OSError:
            pass 

def upload_ss(
    filename, wikifilename, http_host, filedesc,
    statuscallback, errorcallback
):
    """Prepare for server-side upload."""
    statuscallback('Preparing for server-side upload...', -1)

    # Get hash
    md5 = hashlib.md5()
    with open(filename, 'rb') as f:
        while True:
            data = f.read(65536)
            if not data:
                break
            md5.update(data)

    # file name check
    wikifilename = wikifilename.replace('/', '-').replace(' ', '_')
    wikifilename = wikifilename.replace('\r\n', '_')
    wikifilename = wikifilename.replace('\r', '_').replace('\n', '_')

    newfilename = '/srv/v2c/ssu/' + wikifilename
    shutil.move(filename, newfilename)

    with open(newfilename + '.txt', 'w') as filedescfile:
        filedesc = filedesc.replace(
            '[[Category:Uploaded with video2commons]]',
            '[[Category:Uploaded with video2commons/Server-side uploads]]'
        )
        filedescfile.write(filedesc.encode('utf-8'))

    fileurl = 'https://' + http_host + '/' + wikifilename

    raise NeedServerSideUpload(fileurl, md5.hexdigest()) 

def renames(old, new):
    """Like os.renames(), but handles renaming across devices."""
    # Implementation borrowed from os.renames().
    head, tail = os.path.split(new)
    if head and tail and not os.path.exists(head):
        os.makedirs(head)

    shutil.move(old, new)

    head, tail = os.path.split(old)
    if head and tail:
        try:
            os.removedirs(head)
        except OSError:
            pass 

def correct_build_location(self):
        """If the build location was a temporary directory, this will move it
        to a new more permanent location"""
        if self.source_dir is not None:
            return
        assert self.req is not None
        assert self._temp_build_dir
        old_location = self._temp_build_dir
        new_build_dir = self._ideal_build_dir
        del self._ideal_build_dir
        if self.editable:
            name = self.name.lower()
        else:
            name = self.name
        new_location = os.path.join(new_build_dir, name)
        if not os.path.exists(new_build_dir):
            logger.debug('Creating directory %s' % new_build_dir)
            _make_build_dir(new_build_dir)
        if os.path.exists(new_location):
            raise InstallationError(
                'A package already exists in %s; please remove it to continue'
                % display_path(new_location))
        logger.debug('Moving package %s from %s to new location %s'
                     % (self, display_path(old_location), display_path(new_location)))
        shutil.move(old_location, new_location)
        self._temp_build_dir = new_location
        self.source_dir = new_location
        self._egg_info_path = None 

def archive(self, build_dir):
        assert self.source_dir
        create_archive = True
        archive_name = '%s-%s.zip' % (self.name, self.installed_version)
        archive_path = os.path.join(build_dir, archive_name)
        if os.path.exists(archive_path):
            response = ask_path_exists(
                'The file %s exists. (i)gnore, (w)ipe, (b)ackup ' %
                display_path(archive_path), ('i', 'w', 'b'))
            if response == 'i':
                create_archive = False
            elif response == 'w':
                logger.warn('Deleting %s' % display_path(archive_path))
                os.remove(archive_path)
            elif response == 'b':
                dest_file = backup_dir(archive_path)
                logger.warn('Backing up %s to %s'
                            % (display_path(archive_path), display_path(dest_file)))
                shutil.move(archive_path, dest_file)
        if create_archive:
            zip = zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED)
            dir = os.path.normcase(os.path.abspath(self.source_dir))
            for dirpath, dirnames, filenames in os.walk(dir):
                if 'pip-egg-info' in dirnames:
                    dirnames.remove('pip-egg-info')
                for dirname in dirnames:
                    dirname = os.path.join(dirpath, dirname)
                    name = self._clean_zip_name(dirname, dir)
                    zipdir = zipfile.ZipInfo(self.name + '/' + name + '/')
                    zipdir.external_attr = 0x1ED << 16 # 0o755
                    zip.writestr(zipdir, '')
                for filename in filenames:
                    if filename == PIP_DELETE_MARKER_FILENAME:
                        continue
                    filename = os.path.join(dirpath, filename)
                    name = self._clean_zip_name(filename, dir)
                    zip.write(filename, self.name + '/' + name)
            zip.close()
            logger.indent -= 2
            logger.notify('Saved %s' % display_path(archive_path)) 

def install_egg(self, egg_path, tmpdir):
        destination = os.path.join(self.install_dir,os.path.basename(egg_path))
        destination = os.path.abspath(destination)
        if not self.dry_run:
            ensure_directory(destination)

        dist = self.egg_distribution(egg_path)
        if not samefile(egg_path, destination):
            if os.path.isdir(destination) and not os.path.islink(destination):
                dir_util.remove_tree(destination, dry_run=self.dry_run)
            elif os.path.exists(destination):
                self.execute(os.unlink,(destination,),"Removing "+destination)
            uncache_zipdir(destination)
            if os.path.isdir(egg_path):
                if egg_path.startswith(tmpdir):
                    f,m = shutil.move, "Moving"
                else:
                    f,m = shutil.copytree, "Copying"
            elif self.should_unzip(dist):
                self.mkpath(destination)
                f,m = self.unpack_and_compile, "Extracting"
            elif egg_path.startswith(tmpdir):
                f,m = shutil.move, "Moving"
            else:
                f,m = shutil.copy2, "Copying"

            self.execute(f, (egg_path, destination),
                (m+" %s to %s") %
                (os.path.basename(egg_path),os.path.dirname(destination)))

        self.add_output(destination)
        return self.egg_distribution(destination) 

def write_safe(buff, path):
    """ Writes to file in a safe manner, ensuring all or nothing thus barring
    from corruption.

    :param buff: Unicode encoded string.
    :param path: Path to write to
    """
    tmp_fdir, tmp_fpath = os.path.split(path)
    tmp_fpath = ".{0}.new".format(tmp_fpath)
    tmp_fpath = os.path.join(tmp_fdir, tmp_fpath)

    with open(tmp_fpath, 'w') as fh:
        fh.write(buff)

    shutil.move(tmp_fpath, path)  # swap files 

def get_latest_visualized_dataset_folder():
    if hasattr(get_latest_visualized_dataset_folder, "latest_visualized_dataset_folder"):
        return get_latest_visualized_dataset_folder.latest_visualized_dataset_folder
    if os.path.isdir(latest_visualized_dataset_folder):
       shutil.move(
            latest_visualized_dataset_folder,  
            os.path.join(visualized_dataset_folder, "visualized_dataset_folder.old.%s"%datetime.datetime.now().strftime(folder_time_fmt))
        )
        
    os.makedirs(latest_visualized_dataset_folder)
    get_latest_visualized_dataset_folder.latest_visualized_dataset_folder = latest_visualized_dataset_folder
    return latest_visualized_dataset_folder 

def renames(old, new):
    """Like os.renames(), but handles renaming across devices."""
    # Implementation borrowed from os.renames().
    head, tail = os.path.split(new)
    if head and tail and not os.path.exists(head):
        os.makedirs(head)

    shutil.move(old, new)

    head, tail = os.path.split(old)
    if head and tail:
        try:
            os.removedirs(head)
        except OSError:
            pass 

def _correct_build_location(self):
        """Move self._temp_build_dir to self._ideal_build_dir/self.req.name

        For some requirements (e.g. a path to a directory), the name of the
        package is not available until we run egg_info, so the build_location
        will return a temporary directory and store the _ideal_build_dir.

        This is only called by self.egg_info_path to fix the temporary build
        directory.
        """
        if self.source_dir is not None:
            return
        assert self.req is not None
        assert self._temp_build_dir
        assert self._ideal_build_dir
        old_location = self._temp_build_dir
        self._temp_build_dir = None
        new_location = self.build_location(self._ideal_build_dir)
        if os.path.exists(new_location):
            raise InstallationError(
                'A package already exists in %s; please remove it to continue'
                % display_path(new_location))
        logger.debug(
            'Moving package %s from %s to new location %s',
            self, display_path(old_location), display_path(new_location),
        )
        shutil.move(old_location, new_location)
        self._temp_build_dir = new_location
        self._ideal_build_dir = None
        self.source_dir = new_location
        self._egg_info_path = None 

def move_file(src: str, dst: str, rm: bool = True) -> int:
    """
    Copies src into dst
    By default, the src_dir is removed if the copy process succeeds

    :param str src: source directory
    :param str dst: destination directory
    :param bool rm: remove original file Y|N
    :return int: status code:
        - 0: if everything went fine
        - 1: if the destination directory already exists
        - 2: error while moving file
        - 3: file already present
    """
    src = trailing_slash(src)
    dst = trailing_slash(dst)

    if os.path.isdir(src):
        try:
            if src != dst:
                copy_tree(src, dst)
            else:
                return 3

            if rm:
                delete_from_fs(src)  # delete src file
        except FileExistsError:
            msg.Prints.info("ERROR - The file {} already exists".format(dst), "", "", icon=False)
            return 1
    else:
        fname = get_file_name(src)
        try:
            if not dst.endswith(fname):
                shutil.move(src, dst+get_file_name(src))
            else:
                shutil.move(src, dst)
        except shutil.Error:
            return 2
    return 0 

def prepare_metadata_for_build_wheel(self, metadata_directory,
                                         config_settings=None):
        sys.argv = sys.argv[:1] + ['dist_info', '--egg-base',
                                   _to_str(metadata_directory)]
        self.run_setup()

        dist_info_directory = metadata_directory
        while True:
            dist_infos = [f for f in os.listdir(dist_info_directory)
                          if f.endswith('.dist-info')]

            if (len(dist_infos) == 0 and
                len(_get_immediate_subdirectories(dist_info_directory)) == 1):

                dist_info_directory = os.path.join(
                    dist_info_directory, os.listdir(dist_info_directory)[0])
                continue

            assert len(dist_infos) == 1
            break

        # PEP 517 requires that the .dist-info directory be placed in the
        # metadata_directory. To comply, we MUST copy the directory to the root
        if dist_info_directory != metadata_directory:
            shutil.move(
                os.path.join(dist_info_directory, dist_infos[0]),
                metadata_directory)
            shutil.rmtree(dist_info_directory, ignore_errors=True)

        return dist_infos[0] 

def renames(old, new):
    """Like os.renames(), but handles renaming across devices."""
    # Implementation borrowed from os.renames().
    head, tail = os.path.split(new)
    if head and tail and not os.path.exists(head):
        os.makedirs(head)

    shutil.move(old, new)

    head, tail = os.path.split(old)
    if head and tail:
        try:
            os.removedirs(head)
        except OSError:
            pass 

def _correct_build_location(self):
        """Move self._temp_build_dir to self._ideal_build_dir/self.req.name

        For some requirements (e.g. a path to a directory), the name of the
        package is not available until we run egg_info, so the build_location
        will return a temporary directory and store the _ideal_build_dir.

        This is only called by self.egg_info_path to fix the temporary build
        directory.
        """
        if self.source_dir is not None:
            return
        assert self.req is not None
        assert self._temp_build_dir
        assert self._ideal_build_dir
        old_location = self._temp_build_dir
        self._temp_build_dir = None
        new_location = self.build_location(self._ideal_build_dir)
        if os.path.exists(new_location):
            raise InstallationError(
                'A package already exists in %s; please remove it to continue'
                % display_path(new_location))
        logger.debug(
            'Moving package %s from %s to new location %s',
            self, display_path(old_location), display_path(new_location),
        )
        shutil.move(old_location, new_location)
        self._temp_build_dir = new_location
        self._ideal_build_dir = None
        self.source_dir = new_location
        self._egg_info_path = None 

def __init__(self):
        """Initializes the default list of components in Skelebot"""

        self.COMPONENTS = {
            Plugin.__name__.lower(): Plugin,
            Jupyter.__name__.lower(): Jupyter,
            Kerberos.__name__.lower(): Kerberos,
            Bump.__name__.lower(): Bump,
            Prime.__name__.lower(): Prime,
            Dexec.__name__.lower(): Dexec,
            Artifactory.__name__.lower(): Artifactory,
            Registry.__name__.lower(): Registry
        }

        # Add the plugin components to the master list
        pluginsHome = os.path.expanduser(PLUGINS_HOME)
        if (os.path.exists(pluginsHome)):
            sys.path.append(pluginsHome)
            for pluginName in os.listdir(pluginsHome):
                if (pluginName[0] != "_"):
                    try:
                        module = importlib.import_module("{name}.{name}".format(name=pluginName))
                        plugin = getattr(module, pluginName[0].upper() + pluginName[1:])
                        self.COMPONENTS[pluginName.lower()] = plugin
                    except Exception:
                        print(PLUGIN_WARNING.format(pluginName))

                        pluginsQuarantine = os.path.expanduser(PLUGINS_QUARANTINE)
                        if not os.path.exists(pluginsQuarantine):
                            os.makedirs(pluginsQuarantine)

                        src = "{folder}/{name}".format(name=pluginName, folder=pluginsHome)
                        dst = "{folder}/{name}".format(name=pluginName, folder=pluginsQuarantine)
                        shutil.move(src, dst) 

def move(a, b):
    return shutil.move(a, b) 

def ConvertRinex3to2(self):

        # most programs still don't support RINEX 3 (partially implemented in this code)
        # convert to RINEX 2.11 using RinEdit
        cmd = pyRunWithRetry.RunCommand('RinEdit --IF %s --OF %s.t --ver2' % (self.rinex, self.rinex), 15, self.rootdir)

        try:
            out, _ = cmd.run_shell()

            if 'exception' in out.lower():
                raise pyRinexExceptionBadFile('RinEdit returned error converting to RINEX 2.11:\n' + out)

            if not os.path.exists(self.rinex_path + '.t'):
                raise pyRinexExceptionBadFile('RinEdit failed to convert to RINEX 2.11:\n' + out)

            # if all ok, move converted file to rinex_path
            os.remove(self.rinex_path)
            move(self.rinex_path + '.t', self.rinex_path)
            # change version
            self.rinex_version = 2.11

            self.log_event('Origin file was RINEX 3 -> Converted to 2.11')

        except pyRunWithRetry.RunCommandWithRetryExeception as e:
            # catch the timeout except and pass it as a pyRinexException
            raise pyRinexException(str(e)) 

def split_file(self):

        # run in the local folder to get the files inside rootdir
        cmd = pyRunWithRetry.RunCommand('teqc -n_GLONASS 64 -n_GPS 64 -n_Galileo 64 -n_SBAS 64 -tbin 1d rnx ' +
                                        self.rinex, 45, self.rootdir)
        try:
            _, err = cmd.run_shell()
        except pyRunWithRetry.RunCommandWithRetryExeception as e:
            # catch the timeout except and pass it as a pyRinexException
            raise pyRinexException(str(e))

        # successfully binned the file
        # delete current file and rename the new files
        os.remove(self.rinex_path)

        # now we should have as many files named rnxDDD0.??o as days inside the RINEX
        for file in os.listdir(self.rootdir):
            if file[0:3] == 'rnx' and self.identify_type(file) is TYPE_RINEX:
                # rename file
                move(os.path.join(self.rootdir, file), os.path.join(self.rootdir,
                                                                    file.replace('rnx', self.StationCode)))
                # get the info for this file
                try:
                    rnx = ReadRinex(self.NetworkCode, self.StationCode,
                                    os.path.join(self.rootdir, file.replace('rnx', self.StationCode)))
                    # append this rinex object to the multiday list
                    self.multiday_rnx_list.append(rnx)
                except (pyRinexException, pyRinexExceptionBadFile):
                    # there was a problem with one of the multiday files. Do not append
                    pass

        return 

def window_data(self, start=None, end=None, copyto=None):
        """
        Window the RINEX data using TEQC
        :param start: a start datetime or self.firstObs if None
        :param end: a end datetime or self.lastObs if None
        :return:
        """
        if start is None:
            start = self.datetime_firstObs
            self.log_event('Setting start = first obs in window_data')

        if end is None:
            end = self.datetime_lastObs
            self.log_event('Setting end = last obs in window_data')

        cmd = pyRunWithRetry.RunCommand('teqc -n_GLONASS 64 -n_GPS 64 -n_SBAS 64 -n_Galileo 64 -st %i%02i%02i%02i%02i%02i -e %i%02i%02i%02i%02i%02i +obs %s.t %s' % (
                start.year, start.month, start.day, start.hour, start.minute, start.second,
                end.year, end.month, end.day, end.hour, end.minute, end.second, self.rinex_path, self.rinex_path), 5)

        out, err = cmd.run_shell()

        if not 'teqc: failure to read' in str(err):
            # delete the original file and replace with .t
            if copyto is None:
                os.remove(self.rinex_path)
                move(self.rinex_path + '.t', self.rinex_path)
                self.datetime_firstObs = start
                self.datetime_lastObs = end
                self.firstObs = self.datetime_firstObs.strftime('%Y/%m/%d %H:%M:%S')
                self.lastObs = self.datetime_lastObs.strftime('%Y/%m/%d %H:%M:%S')
            else:
                move(self.rinex_path + '.t', copyto)
        else:
            raise pyRinexException(err)

        return 

def decimate(self, decimate_rate, copyto=None):
        # if copy to is passed, then the decimation is done on the copy of the file, not on the current rinex.
        # otherwise, decimation is done in current rinex
        if copyto is not None:
            copyfile(self.rinex_path, copyto)
        else:
            copyto = self.rinex_path
            self.interval = decimate_rate

        if self.rinex_version < 3:
            cmd = pyRunWithRetry.RunCommand('teqc -n_GLONASS 64 -n_GPS 64 -n_SBAS 64 -n_Galileo 64 -O.dec %i '
                                            '+obs %s.t %s' % (decimate_rate, copyto, copyto), 5)
            # leave errors un-trapped on purpose (will raise an error to the parent)
        else:
            cmd = pyRunWithRetry.RunCommand('RinEdit --IF %s --OF %s.t --TN %i --TB %i,%i,%i,%i,%i,%i'
                                            % (os.path.basename(copyto), os.path.basename(copyto), decimate_rate,
                                               self.date.year, self.date.month, self.date.day, 0, 0, 0),
                                            15, self.rootdir)
        out, err = cmd.run_shell()

        if 'teqc: failure to read' not in str(err):
            # delete the original file and replace with .t
            os.remove(copyto)
            move(copyto + '.t', copyto)
        else:
            raise pyRinexException(err)

        self.log_event('RINEX decimated to %is (applied to %s)' % (decimate_rate, str(copyto)))

        return 

def remove_systems(self, systems=('R', 'E', 'S'), copyto=None):
        # if copy to is passed, then the system removal is done on the copy of the file, not on the current rinex.
        # other wise, system removal is done to current rinex
        if copyto is not None:
            copyfile(self.rinex_path, copyto)
        else:
            copyto = self.rinex_path

        if self.rinex_version < 3:
            rsys = '-' + ' -'.join(systems)
            cmd = pyRunWithRetry.RunCommand('teqc -n_GLONASS 64 -n_GPS 64 -n_SBAS 64 -n_Galileo 64 %s +obs %s.t %s' % (rsys, copyto, copyto), 5)
        else:
            rsys = ' --DS '.join(systems)
            cmd = pyRunWithRetry.RunCommand('RinEdit --IF %s --OF %s.t --DS %s' % (os.path.basename(copyto), os.path.basename(copyto), rsys), 15, self.rootdir)

        # leave errors un-trapped on purpose (will raise an error to the parent)
        out, err = cmd.run_shell()

        if not 'teqc: failure to read' in str(err):
            # delete the original file and replace with .t
            os.remove(copyto)
            move(copyto + '.t', copyto)
            # if working on local copy, reload the rinex information
            if copyto == self.rinex_path:
                # reload information from this file
                self.parse_output(self.RunRinSum())
        else:
            raise pyRinexException(err)

        self.log_event('Removed systems %s (applied to %s)' % (','.join(systems), str(copyto)))

        return 

def __add__(self, other):

        if not isinstance(other, ReadRinex):
            raise pyRinexException('type: '+type(other)+' invalid.  Can only splice two RINEX objects.')

        if self.StationCode != other.StationCode:
            raise pyRinexException('Cannot splice together two different stations!')

        # determine which one goes first
        if other.datetime_firstObs > self.datetime_firstObs:
            f1 = self
            f2 = other
        else:
            f1 = other
            f2 = self

        # now splice files
        cmd = pyRunWithRetry.RunCommand('teqc -n_GLONASS 64 -n_GPS 64 -n_SBAS 64 -n_Galileo 64 +obs %s.t %s %s' % (f1.rinex_path, f1.rinex_path, f2.rinex_path), 5)

        # leave errors un-trapped on purpose (will raise an error to the parent)
        out, err = cmd.run_shell()

        if not 'teqc: failure to read' in str(err):
            filename = Utils.move(f1.rinex_path + '.t', f1.rinex_path)
            return ReadRinex(self.NetworkCode, self.StationCode, filename, allow_multiday=True)
        else:
            raise pyRinexException(err) 

def tbin(self):

        # run in the local folder to get the files inside rootdir
        cmd = pyRunWithRetry.RunCommand('teqc -tbin 1d rnx ' + self.rinex, 45, self.rootdir)
        try:
            _, err = cmd.run_shell()
        except pyRunWithRetry.RunCommandWithRetryExeception as e:
            # catch the timeout except and pass it as a pyRinexException
            raise pyRinexException(str(e))
        except Exception:
            raise

        # successfully tbinned the file
        # delete current file and rename the new files
        os.remove(self.rinex_path)

        # now we should have as many files named rnxDDD0.??o as there where inside the RINEX
        for file in os.listdir(self.rootdir):
            if file.endswith('o') and file[0:3] == 'rnx':
                # rename file
                move(os.path.join(self.rootdir,file),os.path.join(self.rootdir,file.replace('rnx', self.StationCode)))
                # get the info for this file
                try:
                    rnx = ReadRinex(self.NetworkCode,self.StationCode,os.path.join(self.rootdir,file.replace('rnx', self.StationCode)))
                    # append this rinex object to the multiday list
                    self.multiday_rnx_list.append(rnx)
                except (pyRinexException, pyRinexExceptionBadFile):
                    # there was a problem with one of the multiday files. Do not append
                    pass

        return 

def download_plugins(data, dest):
    """
    Downloads and extracts plugins

    If the destination directory already exists, the plugin is skipped
    """
    for name, data in data.items():
        version = data['version']
        guid = data['guid']
        url = data['url']
        if 'archive_path' in data:
            archive_path = data['archive_path']
        else:
            archive_path = None

        # Destination directory
        dest_dir = os.path.join(dest, name + '_' + guid)

        if os.path.isdir(dest_dir):
            # If destination path exists, check the version that is installed
            with open(os.path.join(dest_dir, 'manifest.json')) as m:
                data = json.load(m)
                existing_version = data['version']

                # Close json file
                m.close()

                # Version already matches
                if version == existing_version:
                    print('NOTICE: Skipping "{}" download, "{}" already exists'
                          .format(name, version))
                    continue
                # Version does not match, delete old plugin and download
                else:
                    shutil.rmtree(dest_dir)
                    print('Upgrading {} plugin'.format(name))

        # Passed Pre-Download Checks
        print('Downloading "{}" version "{}" ({})'
              .format(name, version, guid))

        # Download zip file into memory
        plugin_url = urlopen(url)
        plugin_zip = zipfile.ZipFile(BytesIO(plugin_url.read()))

        # Extract zip directly into destination if there are no sub-folders
        #  to deal with
        if not archive_path:
            plugin_zip.extractall(path=dest_dir)
        else:
            # Create temporary directory for extraction
            tmp_dir = tempfile.TemporaryDirectory(prefix='galaxy-plugin')

            # Extract zip to temporary directory
            plugin_zip.extractall(path=tmp_dir.name)

            # Move sub-directory into final destination directory
            shutil.move(os.path.join(tmp_dir.name, archive_path), dest_dir)

            # Cleanup temporary directory
            tmp_dir.cleanup() 

def trash(self):
        """Trash item in trash bin.

        Writes .trashinfo file, then moves path to trash bin."""

        # Write .trashinfo file first, so if it fails, we don't trash
        # the file, avoiding "orphaned" files in the bin.

        # Set trashed path (is this the best place to do this?)
        self.trashed_path = self.bin.files_path / self.original_path.name

        log.debug('Preparing to trash "%s" as "%s"', self.original_path, self.trashed_path)

        # Rename if necessary
        self._rename_if_necessary()

        # Set date_trashed
        self.date_trashed = datetime.utcnow()

        # Write trashinfo file
        self._write_trashinfo_file()

        # There seems to be no way to rename a path in Python without
        # potentially overwriting an existing path, so although we
        # change the basename if necessary above, there's still a
        # potential race condition that could cause an existing item
        # in the bin to be overwritten.

        # Actually, it might be possible to make a new hard link to
        # the file inside the trash bin, then unlink the original.
        # But that obviously won't work across filesystems, so maybe
        # it's not worth it.

        # Move path to trash
        try:
            # FIXME: Would Path.rename() suffice here, or does that not work across filesystems?
            shutil.move(self.original_path.as_posix(), self.trashed_path.as_posix())
        except Exception as e:
            log.error('Unable to move item "%s" to trashed path "%s": %s', self.original_path, self.trashed_path, e)

            # FIXME: Remove trashinfo file if trashing fails

            return False
        else:
            log.info('Trashed "%s" as: "%s"', self.original_path, self.trashed_path)

# ** Exceptions 

def download_evozi(driver, num):
    try:
        gp_file = "GooglePlayRank_{num}.txt".format(num=num)
        gp_file_tmp = "GooglePlayRankTmp_{num}.txt".format(num=num)
        with open(gp_file) as f_in:
            pkg = f_in.readline().replace('\n', '').strip()
        # pkg = "adult.coloring.book.mandala.colorfy.coloring.free"
        url = "https://apps.evozi.com/apk-downloader/?id={pkg}"
        _url = url.format(pkg=pkg)
        driver.maximize_window()
        driver.get(_url)
        driver.find_element_by_class_name("btn-lg").click()
        time.sleep(5)
        down_link = driver.find_element_by_class_name("btn-success").get_attribute("href")
        if down_link == _url + "#":
            download_fail_file = "GooglePlayRank_fail_{num}.txt".format(num=num)
            with open(download_fail_file, 'a+', encoding='utf-8') as f:
                _pkg = pkg + "\n"
                f.write(_pkg)
            return
        print(down_link)
        apk_stream = requests.get(down_link, stream=True)
        file_name = pkg + '.apk'
        file_path = os.path.join(os.getcwd(), "apk", file_name)
        with open(file_path, 'wb') as f:
            for chunk in apk_stream.iter_content(chunk_size=512):
                if chunk:
                    f.write(chunk)
        download_success_file = "GooglePlayRank_success_{num}.txt".format(num=num)
        with open(download_success_file, 'a+', encoding='utf-8') as f:
            _pkg = pkg + "\n"
            f.write(_pkg)
        return
    except Exception as e:
        print("download:", e)
    finally:
        with open(gp_file) as f_in:
            with open(gp_file_tmp, 'w') as f_tmp:
                for line in f_in.readlines():
                    if pkg not in line:
                        f_tmp.write(line)
        shutil.move(gp_file_tmp, gp_file)
        print("download:", pkg) 

def _handle_target_dir(self, target_dir, target_temp_dir, upgrade):
        ensure_dir(target_dir)

        # Checking both purelib and platlib directories for installed
        # packages to be moved to target directory
        lib_dir_list = []

        with target_temp_dir:
            # Checking both purelib and platlib directories for installed
            # packages to be moved to target directory
            scheme = distutils_scheme('', home=target_temp_dir.path)
            purelib_dir = scheme['purelib']
            platlib_dir = scheme['platlib']
            data_dir = scheme['data']

            if os.path.exists(purelib_dir):
                lib_dir_list.append(purelib_dir)
            if os.path.exists(platlib_dir) and platlib_dir != purelib_dir:
                lib_dir_list.append(platlib_dir)
            if os.path.exists(data_dir):
                lib_dir_list.append(data_dir)

            for lib_dir in lib_dir_list:
                for item in os.listdir(lib_dir):
                    if lib_dir == data_dir:
                        ddir = os.path.join(data_dir, item)
                        if any(s.startswith(ddir) for s in lib_dir_list[:-1]):
                            continue
                    target_item_dir = os.path.join(target_dir, item)
                    if os.path.exists(target_item_dir):
                        if not upgrade:
                            logger.warning(
                                'Target directory %s already exists. Specify '
                                '--upgrade to force replacement.',
                                target_item_dir
                            )
                            continue
                        if os.path.islink(target_item_dir):
                            logger.warning(
                                'Target directory %s already exists and is '
                                'a link. Pip will not automatically replace '
                                'links, please remove if replacement is '
                                'desired.',
                                target_item_dir
                            )
                            continue
                        if os.path.isdir(target_item_dir):
                            shutil.rmtree(target_item_dir)
                        else:
                            os.remove(target_item_dir)

                    shutil.move(
                        os.path.join(lib_dir, item),
                        target_item_dir
                    ) 

def archive(self, build_dir):
        assert self.source_dir
        create_archive = True
        archive_name = '%s-%s.zip' % (self.name, self.pkg_info()["version"])
        archive_path = os.path.join(build_dir, archive_name)
        if os.path.exists(archive_path):
            response = ask_path_exists(
                'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)bort ' %
                display_path(archive_path), ('i', 'w', 'b', 'a'))
            if response == 'i':
                create_archive = False
            elif response == 'w':
                logger.warning('Deleting %s', display_path(archive_path))
                os.remove(archive_path)
            elif response == 'b':
                dest_file = backup_dir(archive_path)
                logger.warning(
                    'Backing up %s to %s',
                    display_path(archive_path),
                    display_path(dest_file),
                )
                shutil.move(archive_path, dest_file)
            elif response == 'a':
                sys.exit(-1)
        if create_archive:
            zip = zipfile.ZipFile(
                archive_path, 'w', zipfile.ZIP_DEFLATED,
                allowZip64=True
            )
            dir = os.path.normcase(os.path.abspath(self.setup_py_dir))
            for dirpath, dirnames, filenames in os.walk(dir):
                if 'pip-egg-info' in dirnames:
                    dirnames.remove('pip-egg-info')
                for dirname in dirnames:
                    dirname = os.path.join(dirpath, dirname)
                    name = self._clean_zip_name(dirname, dir)
                    zipdir = zipfile.ZipInfo(self.name + '/' + name + '/')
                    zipdir.external_attr = 0x1ED << 16  # 0o755
                    zip.writestr(zipdir, '')
                for filename in filenames:
                    if filename == PIP_DELETE_MARKER_FILENAME:
                        continue
                    filename = os.path.join(dirpath, filename)
                    name = self._clean_zip_name(filename, dir)
                    zip.write(filename, self.name + '/' + name)
            zip.close()
            logger.info('Saved %s', display_path(archive_path)) 

def create_archive(git_path, module, dest, archive, version, repo, result):
    """ Helper function for creating archive using git_archive """
    all_archive_fmt = {'.zip': 'zip', '.gz': 'tar.gz', '.tar': 'tar',
                       '.tgz': 'tgz'}
    _, archive_ext = os.path.splitext(archive)
    archive_fmt = all_archive_fmt.get(archive_ext, None)
    if archive_fmt is None:
        module.fail_json(msg="Unable to get file extension from "
                             "archive file name : %s" % archive,
                         details="Please specify archive as filename with "
                                 "extension. File extension can be one "
                                 "of ['tar', 'tar.gz', 'zip', 'tgz']")

    repo_name = repo.split("/")[-1].replace(".git", "")

    if os.path.exists(archive):
        # If git archive file exists, then compare it with new git archive file.
        # if match, do nothing
        # if does not match, then replace existing with temp archive file.
        tempdir = tempfile.mkdtemp()
        new_archive_dest = os.path.join(tempdir, repo_name)
        new_archive = new_archive_dest + '.' + archive_fmt
        git_archive(git_path, module, dest, new_archive, archive_fmt, version)

        # filecmp is supposed to be efficient than md5sum checksum
        if filecmp.cmp(new_archive, archive):
            result.update(changed=False)
            # Cleanup before exiting
            try:
                shutil.rmtree(tempdir)
            except OSError:
                pass
        else:
            try:
                shutil.move(new_archive, archive)
                shutil.rmtree(tempdir)
                result.update(changed=True)
            except OSError as e:
                module.fail_json(msg="Failed to move %s to %s" %
                                     (new_archive, archive),
                                 details="Error occured while moving : %s"
                                         % to_native(e))
    else:
        # Perform archive from local directory
        git_archive(git_path, module, dest, archive, archive_fmt, version)
        result.update(changed=True)


# =========================================== 

def zip_package(self, module_name, filename, no_pyc):
        orig_filename = filename
        logger.notify('Zip %s (in %s)' % (module_name, display_path(filename)))
        logger.indent += 2
        if filename.endswith('.egg'):
            dest_filename = filename
        else:
            dest_filename = filename + '.zip'
        try:
            ## FIXME: I think this needs to be undoable:
            if filename == dest_filename:
                filename = backup_dir(orig_filename)
                logger.notify('Moving %s aside to %s' % (orig_filename, filename))
                if not self.simulate:
                    shutil.move(orig_filename, filename)
            try:
                logger.info('Creating zip file in %s' % display_path(dest_filename))
                if not self.simulate:
                    zip = zipfile.ZipFile(dest_filename, 'w')
                    zip.writestr(module_name + '/', '')
                    for dirpath, dirnames, filenames in os.walk(filename):
                        if no_pyc:
                            filenames = [f for f in filenames
                                         if not f.lower().endswith('.pyc')]
                        for fns, is_dir in [(dirnames, True), (filenames, False)]:
                            for fn in fns:
                                full = os.path.join(dirpath, fn)
                                dest = os.path.join(module_name, dirpath[len(filename):].lstrip(os.path.sep), fn)
                                if is_dir:
                                    zip.writestr(dest + '/', '')
                                else:
                                    zip.write(full, dest)
                    zip.close()
                logger.info('Removing old directory %s' % display_path(filename))
                if not self.simulate:
                    rmtree(filename)
            except:
                ## FIXME: need to do an undo here
                raise
            ## FIXME: should also be undone:
            self.add_filename_to_pth(dest_filename)
        finally:
            logger.indent -= 2 

def check_destination(self, dest, url, rev_options, rev_display):
        """
        Prepare a location to receive a checkout/clone.

        Return True if the location is ready for (and requires) a
        checkout/clone, False otherwise.
        """
        checkout = True
        prompt = False
        if os.path.exists(dest):
            checkout = False
            if os.path.exists(os.path.join(dest, self.dirname)):
                existing_url = self.get_url(dest)
                if self.compare_urls(existing_url, url):
                    logger.info('%s in %s exists, and has correct URL (%s)' %
                                (self.repo_name.title(), display_path(dest),
                                 url))
                    logger.notify('Updating %s %s%s' %
                                  (display_path(dest), self.repo_name,
                                   rev_display))
                    self.update(dest, rev_options)
                else:
                    logger.warn('%s %s in %s exists with URL %s' %
                                (self.name, self.repo_name,
                                 display_path(dest), existing_url))
                    prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ',
                              ('s', 'i', 'w', 'b'))
            else:
                logger.warn('Directory %s already exists, '
                            'and is not a %s %s.' %
                            (dest, self.name, self.repo_name))
                prompt = ('(i)gnore, (w)ipe, (b)ackup ', ('i', 'w', 'b'))
        if prompt:
            logger.warn('The plan is to install the %s repository %s' %
                        (self.name, url))
            response = ask_path_exists('What to do?  %s' % prompt[0],
                                       prompt[1])

            if response == 's':
                logger.notify('Switching %s %s to %s%s' %
                              (self.repo_name, display_path(dest), url,
                               rev_display))
                self.switch(dest, url, rev_options)
            elif response == 'i':
                # do nothing
                pass
            elif response == 'w':
                logger.warn('Deleting %s' % display_path(dest))
                rmtree(dest)
                checkout = True
            elif response == 'b':
                dest_dir = backup_dir(dest)
                logger.warn('Backing up %s to %s'
                            % (display_path(dest), dest_dir))
                shutil.move(dest, dest_dir)
                checkout = True
        return checkout 

def main():
  cmd_args = argparse.ArgumentParser()
  cmd_args.add_argument("--include-tensorflow", action="store_true")
  cmd_args.add_argument("--output-dir", required=True)
  args = cmd_args.parse_args()
  if not os.path.isdir(args.output_dir):
    raise EnvironmentError(
        "Output directory {} doesn't exist".format(args.output_dir))
  elif not args.output_dir.startswith("/"):
    raise EnvironmentError("Please pass an absolute path to --output-dir.")

  tmp_packaging = tempfile.mkdtemp()
  runfiles, = (path for path in sys.path
               if path.endswith("build_pip_package.runfiles"))

  # Use the dragnn and tensorflow modules to resolve specific paths in the
  # runfiles directory. Current Bazel puts dragnn in a __main__ subdirectory,
  # for example.
  lib_path = os.path.abspath(dragnn.__file__)
  if runfiles not in lib_path:
    raise EnvironmentError("WARNING: Unexpected PYTHONPATH set by Bazel :(")
  base_dir = os.path.dirname(os.path.dirname(lib_path))
  tensorflow_dir = os.path.dirname(tensorflow.__file__)
  if runfiles not in tensorflow_dir:
    raise EnvironmentError("WARNING: Unexpected tf PYTHONPATH set by Bazel :(")

  # Copy the files.
  subprocess.check_call([
      "cp", "-r", os.path.join(base_dir, "dragnn"), os.path.join(
          base_dir, "syntaxnet"), tmp_packaging
  ])
  if args.include_tensorflow:
    subprocess.check_call(
        ["cp", "-r", tensorflow_dir, tmp_packaging])
  shutil.copy(
      os.path.join(base_dir, "dragnn/tools/oss_setup.py"),
      os.path.join(tmp_packaging, "setup.py"))
  subprocess.check_output(
      ["python", "setup.py", "bdist_wheel"], cwd=tmp_packaging)
  wheel, = glob.glob("{}/*.whl".format(os.path.join(tmp_packaging, "dist")))

  shutil.move(wheel, args.output_dir)
  print(
      "Wrote {}".format(os.path.join(args.output_dir, os.path.basename(wheel)))) 

def TrainTestSplit():
    basepath = r'E:\bod-dataset'
    filelist = GetFileFromThisRootDir(os.path.join(basepath, 'images'))
    name = [os.path.basename(os.path.splitext(x)[0]) for x in filelist if (x != 'Thumbs')]
    train_len = int(len(name) * 0.5)
    val_len = int(len(name) * 1 /6)
    test_len = len(name) - train_len - val_len
    print('train_len:', train_len)
    print('val_len:', val_len)
    print('test_len:', test_len)
    random.shuffle(name)
    print('shuffle name:', name)
    train_set= set(name[0:train_len])
    val_set = set(name[train_len:(train_len + val_len)])
    test_set = set(name[(train_len + val_len):])
    print('intersection:', train_set.intersection(test_set))
    imgsrcpath = os.path.join(basepath, 'images')
    txtsrcpath = os.path.join(basepath, 'wordlabel')
    imgtestpath = os.path.join(basepath, 'testset', 'images')
    txttestpath = os.path.join(basepath, 'testset', 'wordlabel')
    imgtrainpath = os.path.join(basepath, 'trainset', 'images')
    txttrainpath = os.path.join(basepath, 'trainset', 'wordlabel')
    imgvalpath = os.path.join(basepath, 'valset', 'images')
    txtvalpath = os.path.join(basepath, 'valset', 'wordlabel')

    #filemove(imgsrcpath, imgtestpath, test_set, '.png')
    filemove(txtsrcpath, txttestpath, test_set, '.txt')

    #filemove(imgsrcpath, imgtrainpath, train_set, '.png')
    filemove(txtsrcpath, txttrainpath, train_set, '.txt')

    #filemove(imgsrcpath, imgvalpath, val_set, '.png')
    filemove(txtsrcpath, txtvalpath, val_set, '.txt')

    # for imgname in train_set:
    #     if (imgname == 'Thumbs'):
    #         continue
    #     srcname = os.path.join(basepath, 'images', imgname + '.tif')
    #     dstname = os.path.join(basepath, 'train', 'images', imgname + '.tif')
    #     shutil.move(srcname, dstname)
    #     srctxt = os.path.join(basepath, 'labelTxt', imgname + '.txt')
    #     dsttxt = os.path.join(basepath, 'train', 'labelTxt', imgname + '.txt')
    #     print(srctxt)
    #     print(dsttxt)
    #     shutil.move(srctxt, dsttxt)
    # for imgname in test_set:
    #     if (imgname == 'Thumbs'):
    #         continue
    #     srcname = os.path.join(basepath, 'images', imgname + '.tif')
    #     dstname = os.path.join(basepath, 'test', 'images', imgname + '.tif')
    #     shutil.move(srcname, dstname)
    #     srctxt = os.path.join(basepath, 'labelTxt', imgname + '.txt')
    #     dsttxt = os.path.join(basepath, 'test', 'labelTxt', imgname + '.txt')
    #     shutil.move(srctxt, dsttxt) 

def toggle_introspection(start, sm=None):
    global rosbag_proc, ac_proc, tmt_proc, sis, ad_proc, ts_proc, goal_proc
    if start:
        toggle_experiment_recording(True)
        if not os.path.isdir(latest_experiment_record_folder):
            os.makedirs(latest_experiment_record_folder)
        rosbag_proc = RosbagProc(
            os.path.join(latest_experiment_record_folder, "record.bag"),
            topics_to_be_recorded_into_rosbag
        )
        rosbag_proc.start()
        tmt_proc = TagMultimodalTopicProc()
        tmt_proc.start()
        sis = smach_ros.IntrospectionServer('MY_SERVER', sm, '/SM_ROOT')
        sis.start()

        if not HUMAN_AS_MODEL_MODE:
            # ad_proc = AnomalyDetectionProc()
            # ad_proc.start()
            # ac_proc = AnomalyClassificationProc()
            # ac_proc.start()
            pass

        # ts_proc = TimeseriesProc()
        # ts_proc.start()
        goal_proc = GoalProc()
        goal_proc.start()
        listen_HMM_anomaly_signal()
    else:
        experiment_name = 'experiment_at_%s'%datetime.datetime.now().strftime(folder_time_fmt)
        toggle_experiment_recording(False, experiment_name)
        if rosbag_proc:
            rospy.loginfo("Tring to tear down rosbag_proc")
            rosbag_proc.stop()
            shutil.move(
                latest_experiment_record_folder, 
                os.path.join(
                    experiment_record_folder,
                    experiment_name
                )
            )
        if ac_proc:
            rospy.loginfo("Tring to tear down ac_proc")
            ac_proc.stop()
        if tmt_proc:
            rospy.loginfo("Tring to tear down tmt_proc")
            tmt_proc.stop()
        if sis:
            rospy.loginfo("Tring to tear down sis")
            sis.stop()
        if ad_proc:
            rospy.loginfo("Tring to tear down ad_proc")
            ad_proc.stop()
        if ts_proc:
            rospy.loginfo("Tring to tear down ts_proc")
            ts_proc.stop()
        if goal_proc:
            rospy.loginfo("Tring to tear down goal_proc")
            goal_proc.stop() 

def archive(self, build_dir):
        assert self.source_dir
        create_archive = True
        archive_name = '%s-%s.zip' % (self.name, self.pkg_info()["version"])
        archive_path = os.path.join(build_dir, archive_name)
        if os.path.exists(archive_path):
            response = ask_path_exists(
                'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)bort ' %
                display_path(archive_path), ('i', 'w', 'b', 'a'))
            if response == 'i':
                create_archive = False
            elif response == 'w':
                logger.warning('Deleting %s', display_path(archive_path))
                os.remove(archive_path)
            elif response == 'b':
                dest_file = backup_dir(archive_path)
                logger.warning(
                    'Backing up %s to %s',
                    display_path(archive_path),
                    display_path(dest_file),
                )
                shutil.move(archive_path, dest_file)
            elif response == 'a':
                sys.exit(-1)
        if create_archive:
            zip = zipfile.ZipFile(
                archive_path, 'w', zipfile.ZIP_DEFLATED,
                allowZip64=True
            )
            dir = os.path.normcase(os.path.abspath(self.setup_py_dir))
            for dirpath, dirnames, filenames in os.walk(dir):
                if 'pip-egg-info' in dirnames:
                    dirnames.remove('pip-egg-info')
                for dirname in dirnames:
                    dirname = os.path.join(dirpath, dirname)
                    name = self._clean_zip_name(dirname, dir)
                    zipdir = zipfile.ZipInfo(self.name + '/' + name + '/')
                    zipdir.external_attr = 0x1ED << 16  # 0o755
                    zip.writestr(zipdir, '')
                for filename in filenames:
                    if filename == PIP_DELETE_MARKER_FILENAME:
                        continue
                    filename = os.path.join(dirpath, filename)
                    name = self._clean_zip_name(filename, dir)
                    zip.write(filename, self.name + '/' + name)
            zip.close()
            logger.info('Saved %s', display_path(archive_path)) 

def install_egg(self, egg_path, tmpdir):
        destination = os.path.join(
            self.install_dir,
            os.path.basename(egg_path),
        )
        destination = os.path.abspath(destination)
        if not self.dry_run:
            ensure_directory(destination)

        dist = self.egg_distribution(egg_path)
        if not samefile(egg_path, destination):
            if os.path.isdir(destination) and not os.path.islink(destination):
                dir_util.remove_tree(destination, dry_run=self.dry_run)
            elif os.path.exists(destination):
                self.execute(
                    os.unlink,
                    (destination,),
                    "Removing " + destination,
                )
            try:
                new_dist_is_zipped = False
                if os.path.isdir(egg_path):
                    if egg_path.startswith(tmpdir):
                        f, m = shutil.move, "Moving"
                    else:
                        f, m = shutil.copytree, "Copying"
                elif self.should_unzip(dist):
                    self.mkpath(destination)
                    f, m = self.unpack_and_compile, "Extracting"
                else:
                    new_dist_is_zipped = True
                    if egg_path.startswith(tmpdir):
                        f, m = shutil.move, "Moving"
                    else:
                        f, m = shutil.copy2, "Copying"
                self.execute(
                    f,
                    (egg_path, destination),
                    (m + " %s to %s") % (
                        os.path.basename(egg_path),
                        os.path.dirname(destination)
                    ),
                )
                update_dist_caches(
                    destination,
                    fix_zipimporter_caches=new_dist_is_zipped,
                )
            except Exception:
                update_dist_caches(destination, fix_zipimporter_caches=False)
                raise

        self.add_output(destination)
        return self.egg_distribution(destination) 

def install_egg(self, egg_path, tmpdir):
        destination = os.path.join(
            self.install_dir,
            os.path.basename(egg_path),
        )
        destination = os.path.abspath(destination)
        if not self.dry_run:
            ensure_directory(destination)

        dist = self.egg_distribution(egg_path)
        if not samefile(egg_path, destination):
            if os.path.isdir(destination) and not os.path.islink(destination):
                dir_util.remove_tree(destination, dry_run=self.dry_run)
            elif os.path.exists(destination):
                self.execute(
                    os.unlink,
                    (destination,),
                    "Removing " + destination,
                )
            try:
                new_dist_is_zipped = False
                if os.path.isdir(egg_path):
                    if egg_path.startswith(tmpdir):
                        f, m = shutil.move, "Moving"
                    else:
                        f, m = shutil.copytree, "Copying"
                elif self.should_unzip(dist):
                    self.mkpath(destination)
                    f, m = self.unpack_and_compile, "Extracting"
                else:
                    new_dist_is_zipped = True
                    if egg_path.startswith(tmpdir):
                        f, m = shutil.move, "Moving"
                    else:
                        f, m = shutil.copy2, "Copying"
                self.execute(
                    f,
                    (egg_path, destination),
                    (m + " %s to %s") % (
                        os.path.basename(egg_path),
                        os.path.dirname(destination)
                    ),
                )
                update_dist_caches(
                    destination,
                    fix_zipimporter_caches=new_dist_is_zipped,
                )
            except Exception:
                update_dist_caches(destination, fix_zipimporter_caches=False)
                raise

        self.add_output(destination)
        return self.egg_distribution(destination) 

def archive(self, build_dir):
        assert self.source_dir
        create_archive = True
        archive_name = '%s-%s.zip' % (self.name, self.pkg_info()["version"])
        archive_path = os.path.join(build_dir, archive_name)
        if os.path.exists(archive_path):
            response = ask_path_exists(
                'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)bort ' %
                display_path(archive_path), ('i', 'w', 'b', 'a'))
            if response == 'i':
                create_archive = False
            elif response == 'w':
                logger.warning('Deleting %s', display_path(archive_path))
                os.remove(archive_path)
            elif response == 'b':
                dest_file = backup_dir(archive_path)
                logger.warning(
                    'Backing up %s to %s',
                    display_path(archive_path),
                    display_path(dest_file),
                )
                shutil.move(archive_path, dest_file)
            elif response == 'a':
                sys.exit(-1)
        if create_archive:
            zip = zipfile.ZipFile(
                archive_path, 'w', zipfile.ZIP_DEFLATED,
                allowZip64=True
            )
            dir = os.path.normcase(os.path.abspath(self.setup_py_dir))
            for dirpath, dirnames, filenames in os.walk(dir):
                if 'pip-egg-info' in dirnames:
                    dirnames.remove('pip-egg-info')
                for dirname in dirnames:
                    dirname = os.path.join(dirpath, dirname)
                    name = self._clean_zip_name(dirname, dir)
                    zipdir = zipfile.ZipInfo(self.name + '/' + name + '/')
                    zipdir.external_attr = 0x1ED << 16  # 0o755
                    zip.writestr(zipdir, '')
                for filename in filenames:
                    if filename == PIP_DELETE_MARKER_FILENAME:
                        continue
                    filename = os.path.join(dirpath, filename)
                    name = self._clean_zip_name(filename, dir)
                    zip.write(filename, self.name + '/' + name)
            zip.close()
            logger.info('Saved %s', display_path(archive_path)) 

def install_egg(self, egg_path, tmpdir):
        destination = os.path.join(
            self.install_dir,
            os.path.basename(egg_path),
        )
        destination = os.path.abspath(destination)
        if not self.dry_run:
            ensure_directory(destination)

        dist = self.egg_distribution(egg_path)
        if not samefile(egg_path, destination):
            if os.path.isdir(destination) and not os.path.islink(destination):
                dir_util.remove_tree(destination, dry_run=self.dry_run)
            elif os.path.exists(destination):
                self.execute(
                    os.unlink,
                    (destination,),
                    "Removing " + destination,
                )
            try:
                new_dist_is_zipped = False
                if os.path.isdir(egg_path):
                    if egg_path.startswith(tmpdir):
                        f, m = shutil.move, "Moving"
                    else:
                        f, m = shutil.copytree, "Copying"
                elif self.should_unzip(dist):
                    self.mkpath(destination)
                    f, m = self.unpack_and_compile, "Extracting"
                else:
                    new_dist_is_zipped = True
                    if egg_path.startswith(tmpdir):
                        f, m = shutil.move, "Moving"
                    else:
                        f, m = shutil.copy2, "Copying"
                self.execute(
                    f,
                    (egg_path, destination),
                    (m + " %s to %s") % (
                        os.path.basename(egg_path),
                        os.path.dirname(destination)
                    ),
                )
                update_dist_caches(
                    destination,
                    fix_zipimporter_caches=new_dist_is_zipped,
                )
            except Exception:
                update_dist_caches(destination, fix_zipimporter_caches=False)
                raise

        self.add_output(destination)
        return self.egg_distribution(destination) 

def Database1(key,code,mypath): #搜尋JAVBUS
	global dirpath,title,coverurl
	url = "https://www.javbus.com/"+code
	response = requests.get(url,headers = {'User-Agent':ua.random})
	response.encoding = 'UTF-8' 
	soup = BeautifulSoup(response.text, 'lxml')

	if soup.find("title").getText() == "404 Not Found" or soup.find("title").getText() == "404 Page Not Found! - JavBus":
		return {'success':False,'error':code+" 404 Not Found"}
	elif soup.find("h3") == None:
		return {'success':False,'error':code+" Unknown Error"}
		
	article = soup.find("div", {"class": "container"})
	if article == None:
		return {'success':False,'error':code+" Unknown Error"}

	title = article.find("h3").getText()
	coverurl = article.find("a", {"class": "bigImage"}).get("href")
	allinfo = article.find("div",{"class":"col-md-3 info"}).find_all("p")
	code,date,time,dierector,producer,pulisher,series,genre,actress,allpreview="","","","","","","","","",[]
	if article.find("div",{"id":"sample-waterfall"}):
		waterfall = article.find("div",{"id":"sample-waterfall"}).find_all("a",{"class":"sample-box"})
		allpreview = [prev.get("href").strip() for prev in waterfall]
	for nfo in range(len(allinfo)):
		if allinfo[nfo].getText().split(" ")[0] == "識別碼:":
			code = allinfo[nfo].getText().split(" ")[1].strip()
		elif allinfo[nfo].getText().split(" ")[0] == "發行日期:":
			date = allinfo[nfo].getText().split(" ")[1].strip()
		elif allinfo[nfo].getText().split(" ")[0] == "長度:":
			time = allinfo[nfo].getText().split(" ")[1].strip()
		elif allinfo[nfo].getText().split(" ")[0] == "導演:":
			dierector = allinfo[nfo].getText().split(" ")[1].strip()
		elif allinfo[nfo].getText().split(" ")[0] == "製作商:":
			producer = allinfo[nfo].getText().split(" ")[1].strip()
		elif allinfo[nfo].getText().split(" ")[0] == "發行商:":
			pulisher = allinfo[nfo].getText().split(" ")[1].strip()
		elif allinfo[nfo].getText().split(" ")[0] == "系列:":
			series = allinfo[nfo].getText().split(" ")[1].strip()
		elif allinfo[nfo].getText() == "類別:":
			genre = [g.getText().strip() for g in allinfo[nfo+1].find_all("span",{"class":"genre"}) ]
		elif allinfo[nfo].getText() == "演員:":
			if nfo+1 < len(allinfo):
				actress = [g.getText().strip() for g in allinfo[nfo+1].find_all("span",{"class":"genre"}) ]

	Sort2Dir(key,code,mypath)
	os.chdir(mypath)
	mergename = code+"_preview.jpg"
	mergepath = config.tempfolder+"\\"+code
	if not os.path.isfile(dirpath+"\\"+mergename) and len(allpreview)>0:
		Merge(code,allpreview,tempfolder=config.tempfolder,signpic=config.signpic)
		shutil.move(mergepath+"\\"+mergename,dirpath+"\\"+mergename) #Move
		#shutil.rmtree(mergepath) #清除Cache
	save = [code,title.replace(code,'').strip(),series,",".join(actress),",".join(genre),date,time,dierector,producer,pulisher]
	return {'success':True,'dirpath':dirpath,'code':code,'save':save,'title':title.replace(code,'').strip()} 

def Database2(key,code,mypath): #搜尋JAV321
	global dirpath,title,coverurl
	surl = "https://www.jav321.com/search"
	payload = {'sn': code}
	response = requests.post(url=surl, data=payload, headers={'User-Agent':ua.random}) 
	response.encoding = 'UTF-8' 
	soup = BeautifulSoup(response.text, 'lxml')
	if soup.find("div", {"class": "alert"}):
		return {'success':False,'error':soup.find("div", {"class": "alert"}).getText()}

	t1 = soup.find("h3").getText()
	t2 = soup.find("h3").find("small").getText()
	title = code + " " +t1.replace(t2,"").strip()
	imgs = soup.find_all("div","col-xs-12 col-md-12")[:-1]
	imglist = [i.find("img").get("src") for i in imgs]
	coverurl = imglist[0]
	allpreview = imglist[1:]

	allinfo = soup.find("div",{"class":"col-md-9"})
	allinfo = str(allinfo).split("<br/>")

	actress,producer,genre,code,date,time,series="","","","","","",""
	for nfo in allinfo:
		nfo2 = BeautifulSoup(nfo, 'lxml').getText()
		if "女优:" in nfo2:
			actress = nfo2.replace("女优:","").strip().split("   ")
		elif "片商:" in nfo2:
			producer = nfo2.replace("片商:","").strip()
		elif "标签:" in nfo2:
			genre = nfo2.replace("标签:","").strip().split(" ")
		elif "番号:" in nfo2:
			code = nfo2.replace("番号:","").strip().upper()
			code = key+code[code.find("-"):]
		elif "发行日期:" in nfo2:
			date = nfo2.replace("发行日期:","").strip()
		elif "播放时长:" in nfo2:
			time = nfo2.replace("播放时长:","").strip().replace("分钟","分鐘")
		elif "系列:" in nfo2:
			series = nfo2.replace("系列:","").strip()
 
	Sort2Dir(key,code,mypath)
	os.chdir(mypath)
	mergename = code+"_preview.jpg"
	mergepath = config.tempfolder+"\\"+code
	if not os.path.isfile(dirpath+"\\"+mergename) and len(allpreview)>0:
		Merge(code,allpreview,tempfolder=config.tempfolder,signpic=config.signpic)
		shutil.move(mergepath+"\\"+mergename,dirpath+"\\"+mergename) #Move
		#shutil.rmtree(mergepath) #清除Cache
	save = [code,title.replace(code,'').strip(),series,",".join(actress),",".join(genre),date,time,'',producer,'']
	return {'success':True,'dirpath':dirpath,'code':code,'save':save,'title':title.replace(code,'').strip()}

#shutil.rmtree(config.tempfolder) #清除Cache

#Test 

def prepare_luna():
    luna_raw = config['luna_raw']
    luna_abbr = config['luna_abbr']
    luna_data = config['luna_data']
    #luna_segment = config['luna_segment']
    finished_flag = '.flag_prepareluna'

    if not os.path.exists(finished_flag):
        print('start changing luna name')
        subsetdirs = [os.path.join(luna_raw, f) for f in os.listdir(luna_raw) if
                      f.startswith('subset') and os.path.isdir(os.path.join(luna_raw, f))]
        if not os.path.exists(luna_data):
            os.mkdir(luna_data)

        abbrevs = np.array(pandas.read_csv(config['luna_abbr'], header=None))
        namelist = list(abbrevs[:, 1])
        ids = abbrevs[:, 0]

        for d in subsetdirs:
            files = os.listdir(d)
            files.sort()

            for f in files:
                name = f[:-4]
                id = ids[namelist.index(name)]
                filename = '0' * (3 - len(str(id))) + str(id)
                shutil.move(os.path.join(d, f), os.path.join(luna_data, filename + f[-4:]))
                print(os.path.join(luna_data, str(id) + f[-4:]))

        files = [f for f in os.listdir(luna_data) if f.endswith('mhd')]
        for file in files:
            with open(os.path.join(luna_data, file), 'r') as f:
                content = f.readlines()
                id = file.split('.mhd')[0]
                filename = '0' * (3 - len(str(id))) + str(id)
                content[-1] = 'ElementDataFile = ' + filename + '.raw\n'
                print(content[-1])
            with open(os.path.join(luna_data, file), 'w') as f:
                f.writelines(content)

    print('end changing luna name')
    f = open(finished_flag, "w+") 

def rename(self, new_name=None, NetworkCode=None, StationCode=None):

        # function that renames the local crinez and rinex file based on the provided information
        # it also changes the variables in the object to reflect this change
        # new name can be any valid format (??d.Z, .??o, ??d, ??o.Z)

        if new_name:
            rinex = os.path.basename(self.to_format(new_name, TYPE_RINEX))
            # do not continue executing unless there is a REAL change!
            if rinex != self.rinex:
                crinez = os.path.basename(self.to_format(new_name, TYPE_CRINEZ))

                # rename the rinex
                if os.path.isfile(self.rinex_path):
                    move(self.rinex_path, os.path.join(self.rootdir, rinex))

                self.rinex_path = os.path.join(self.rootdir, rinex)

                # rename the files
                # check if local crinez exists (possibly made by compress_local_copyto)
                if os.path.isfile(self.crinez_path):
                    move(self.crinez_path, os.path.join(self.rootdir, crinez))

                self.crinez_path = os.path.join(self.rootdir, crinez)

                # rename the local copy of the origin file (if exists)
                # only cases that need to be renamed (again, IF they exist; they shouldn't, but just in case)
                # are RINEZ and CRINEX since RINEX and CRINEZ are renamed above
                if os.path.isfile(self.local_copy):
                    if self.origin_type is TYPE_RINEZ:
                        local = os.path.basename(self.to_format(new_name, TYPE_RINEZ))
                        move(self.local_copy, os.path.join(self.rootdir, local))
                    elif self.origin_type is TYPE_CRINEX:
                        local = os.path.basename(self.to_format(new_name, TYPE_CRINEX))
                        move(self.local_copy, os.path.join(self.rootdir, local))

                self.crinez = crinez
                self.rinex = rinex

                self.log_event('RINEX/CRINEZ renamed to %s' % rinex)

                # update the database dictionary record
                self.record['Filename'] = self.rinex

        # we don't touch the metadata StationCode and NetworkCode unless explicitly passed
        if NetworkCode:
            self.NetworkCode = NetworkCode.strip().lower()
            self.record['NetworkCode'] = NetworkCode.strip().lower()

        if StationCode:
            self.StationCode = StationCode.strip().lower()
            self.record['StationCode'] = StationCode.strip().lower()

        return