Python ujson.load() Examples

def create_language_name_map(self) -> None:
        join = os.path.join
        deploy_root = settings.DEPLOY_ROOT
        path = join(deploy_root, 'locale', 'language_options.json')
        output_path = join(deploy_root, 'locale', 'language_name_map.json')

        with open(path) as reader:
            languages = ujson.load(reader)
            lang_list = []
            for lang_info in languages['languages']:
                lang_info['name'] = lang_info['name_local']
                del lang_info['name_local']
                lang_list.append(lang_info)

            lang_list.sort(key=lambda lang: lang['name'])

        with open(output_path, 'w') as output_file:
            ujson.dump({'name_map': lang_list}, output_file, indent=4, sort_keys=True)
            output_file.write('\n') 

def __init__(self, config=None):

		if isinstance(config, dict) or isinstance(config, OrderedDict):
			self.config = config
		elif isinstance(config, str):
			try:
				self.config = json.load(open(config, "r"))
			except:
				self.config = {}

		self.username = self.config.get("username", "data_security_es_45")
		self.password = self.config.get("password", "Nb6121ca7ffe3")
		es_url = self.config.get("es_url", ['http://zsearch.alipay.com:9999'])

		if isinstance(es_url, list):
			self.es_url = es_url
		else:
			self.es_url = [es_url]

		self.es = Elasticsearch(self.es_url, http_auth=(self.username, self.password)) 

def export_usermessages_batch(input_path: Path, output_path: Path,
                              consent_message_id: Optional[int]=None) -> None:
    """As part of the system for doing parallel exports, this runs on one
    batch of Message objects and adds the corresponding UserMessage
    objects. (This is called by the export_usermessage_batch
    management command)."""
    with open(input_path) as input_file:
        output = ujson.load(input_file)
    message_ids = [item['id'] for item in output['zerver_message']]
    user_profile_ids = set(output['zerver_userprofile_ids'])
    del output['zerver_userprofile_ids']
    realm = Realm.objects.get(id=output['realm_id'])
    del output['realm_id']
    output['zerver_usermessage'] = fetch_usermessages(realm, set(message_ids), user_profile_ids,
                                                      output_path, consent_message_id)
    write_message_export(output_path, output)
    os.unlink(input_path) 

def read_metric(path: str) -> List[int]:
    """
    Read metric file in JSON or Rdump format.
    Return dimensions of entry "inv_metric".
    """
    if path.endswith('.json'):
        with open(path, 'r') as fd:
            metric_dict = json.load(fd)
        if 'inv_metric' in metric_dict:
            dims = np.asarray(metric_dict['inv_metric'])
            return list(dims.shape)
        else:
            raise ValueError(
                'metric file {}, bad or missing'
                ' entry "inv_metric"'.format(path)
            )
    else:
        dims = list(read_rdump_metric(path))
        if dims is None:
            raise ValueError(
                'metric file {}, bad or missing'
                ' entry "inv_metric"'.format(path)
            )
        return dims 

def Apps(self, data=None):
        self.destroy()
        self.create_window(show=False)
        self.btngroup = ButtonGroup(self.window, 40, 30, 240, 40, 10)
        self.widgets.append(self.btngroup)
        for app in uos.listdir('/apps'):
            try:
                with open('/apps/{}/app.json'.format(app)) as fp:
                    data = json.load(fp)
            except Exception as e:
                print(e)
                continue
            if 'name' in data:
                self.btngroup.add(data['name'], self.run_app, data=app)
        self.btngroup.end()
        self.window.show() 

def configure(ctx):
    """
    Configure the following application settings:

    (1) Default encoding to load files.
    (2) HTTP/HTTPS proxy server URI (for url sub-command).

    Configurations are written to '~/.sqlitebiter'.
    You can remove these settings by deleting '~/.sqlitebiter'.
    """

    initialize_logger("{:s} file".format(PROGRAM_NAME), ctx.obj[Context.LOG_LEVEL])

    logger.debug("{} configuration file existence: {}".format(PROGRAM_NAME, app_config_mgr.exists))

    sys.exit(app_config_mgr.configure()) 

def process_file(filename, config, word_counter=None, char_counter=None):
    data = json.load(open(filename, 'r'))

    examples = []
    eval_examples = {}

    outputs = Parallel(n_jobs=12, verbose=10)(delayed(_process_article)(article, config) for article in data)
    # outputs = [_process_article(article, config) for article in data]
    examples = [e[0] for e in outputs]
    for _, e in outputs:
        if e is not None:
            eval_examples[e['id']] = e

    # only count during training
    if word_counter is not None and char_counter is not None:
        for example in examples:
            for token in example['ques_tokens'] + example['context_tokens']:
                word_counter[token] += 1
                for char in token:
                    char_counter[char] += 1

    random.shuffle(examples)
    print("{} questions in total".format(len(examples)))

    return examples, eval_examples 

def populate_new_fields(apps: StateApps, schema_editor: DatabaseSchemaEditor) -> None:
    # Open the JSON file which contains the data to be used for migration.
    MIGRATION_DATA_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), "management", "data")
    path_to_unified_reactions = os.path.join(MIGRATION_DATA_PATH, "unified_reactions.json")
    unified_reactions = ujson.load(open(path_to_unified_reactions))

    Reaction = apps.get_model('zerver', 'Reaction')
    for reaction in Reaction.objects.all():
        reaction.emoji_code = unified_reactions.get(reaction.emoji_name)
        if reaction.emoji_code is None:
            # If it's not present in the unified_reactions map, it's a realm emoji.
            reaction.emoji_code = reaction.emoji_name
            if reaction.emoji_name == 'zulip':
                # `:zulip:` emoji is a zulip special custom emoji.
                reaction.reaction_type = 'zulip_extra_emoji'
            else:
                reaction.reaction_type = 'realm_emoji'
        reaction.save() 

def load_answers(filename):
    """Load the answers only of a SQuAD dataset. Store as qid -> [answers]."""
    # Load JSON file
    with open(filename) as f:
        examples = json.load(f)['data']

    ans = {}
    for article in examples:
        for paragraph in article['paragraphs']:
            for qa in paragraph['qas']:
                ans[qa['id']] = list(map(lambda x: x['text'], qa['answers']))
    return ans


# ------------------------------------------------------------------------------
# Dictionary building
# ------------------------------------------------------------------------------ 

def team_view(request: HttpRequest) -> HttpResponse:
    if not settings.ZILENCER_ENABLED:
        return HttpResponseRedirect('https://zulip.com/team/', status=301)

    try:
        with open(settings.CONTRIBUTOR_DATA_FILE_PATH) as f:
            data = ujson.load(f)
    except FileNotFoundError:
        data = {'contrib': {}, 'date': "Never ran."}

    return TemplateResponse(
        request,
        'zerver/team.html',
        context={
            'page_params': {
                'contrib': data['contrib'],
            },
            'date': data['date'],
        },
    ) 

def load_dataset(path):
    """Load json file and store fields separately."""
    with open(path) as f:
        data = json.load(f)['data']
    output = {'qids': [], 'questions': [], 'answers': [],
              'contexts': [], 'qid2cid': []}
    for article in data:
        for paragraph in article['paragraphs']:
            output['contexts'].append(paragraph['context'])
            for qa in paragraph['qas']:
                output['qids'].append(qa['id'])
                output['questions'].append(qa['question'])
                output['qid2cid'].append(len(output['contexts']) - 1)
                if 'answers' in qa:
                    output['answers'].append(qa['answers'])
    return output 

def load_dataset(filename):
    with codecs.open(filename, mode='r', encoding='utf-8') as f:
        dataset = ujson.load(f)
    return dataset 

def test_export_single_user(self) -> None:
        output_dir = self._make_output_dir()
        cordelia = self.example_user('cordelia')

        with patch('logging.info'):
            do_export_user(cordelia, output_dir)

        def read_file(fn: str) -> Any:
            full_fn = os.path.join(output_dir, fn)
            with open(full_fn) as f:
                return ujson.load(f)

        messages = read_file('messages-000001.json')
        user = read_file('user.json')

        exported_user_id = self.get_set(user['zerver_userprofile'], 'id')
        self.assertEqual(exported_user_id, {cordelia.id})
        exported_user_email = self.get_set(user['zerver_userprofile'], 'email')
        self.assertEqual(exported_user_email, {cordelia.email})

        exported_recipient_type_id = self.get_set(user['zerver_recipient'], 'type_id')
        self.assertIn(cordelia.id, exported_recipient_type_id)

        exported_stream_id = self.get_set(user['zerver_stream'], 'id')
        self.assertIn(list(exported_stream_id)[0], exported_recipient_type_id)

        exported_recipient_id = self.get_set(user['zerver_recipient'], 'id')
        exported_subscription_recipient = self.get_set(user['zerver_subscription'], 'recipient')
        self.assertEqual(exported_recipient_id, exported_subscription_recipient)

        exported_messages_recipient = self.get_set(messages['zerver_message'], 'recipient')
        self.assertIn(list(exported_messages_recipient)[0], exported_recipient_id) 

def load_tcconfig(self, config_file_path):
        from voluptuous import Schema, Required, Any, ALLOW_EXTRA

        schema = Schema(
            {Required(str): {Any(*TrafficDirection.LIST): {str: {str: Any(str, int, float)}}}},
            extra=ALLOW_EXTRA,
        )

        with open(config_file_path, encoding="utf-8") as fp:
            self.__config_table = json.load(fp)

        schema(self.__config_table)
        self.__logger.debug(
            "tc config file: {:s}".format(json.dumps(self.__config_table, indent=4))
        ) 

def name(self) -> Union[str, None]:
        global _profile_icon_names
        if _profile_icon_names is None:
            module_directory = os.path.dirname(os.path.realpath(__file__))
            module_directory, _ = os.path.split(module_directory)  # Go up one directory
            module_directory, _ = os.path.split(module_directory)  # Go up another directory
            filename = os.path.join(module_directory, "profile_icon_names.json")
            with open(filename) as f:
                _profile_icon_names = json.load(f)
            _profile_icon_names = {int(key): value for key, value in _profile_icon_names.items()}
        try:
            return _profile_icon_names[self._data[ProfileIconData].id] or None
        except KeyError:
            return None 

def get_hashes(path):
    # Recursivly search for a file to hash in path
    print("Searching:", path)

    c1 = ["rclone", "lsjson", "--files-only", "--hash", "--copy-links", path]
    c2 = ["rclone", "lsjson", "--dirs-only", "--copy-links", path]

    r1 = subprocess.Popen(c1, stdout=subprocess.PIPE)
    r2 = subprocess.Popen(c2, stdout=subprocess.PIPE)

    files = ujson.load(r1.stdout)
    dirs = ujson.load(r2.stdout)

    if len(files) == 0:
        for d in dirs:
            tmp = get_hashes(os.path.join(path, d["Path"]))
            if tmp is not None:
                return tmp

        return None
    else:
        if "Hashes" in files[0]:
            return set(files[0]["Hashes"].keys())
        else:
            print(
                "Something seems to have gone wrong, please raise an issue on github and include this traceback"
            )
            print(files) 

def read(file):
    """Reads json do dict and returns dict."""
    try:
        with open(file, "r") as fp:
            d = ujson.load(fp)
            if not isinstance(d, dict):
                raise ValueError("old file format")
    except Exception as e:
        emsg = "{} is corrupt ({}). ".format(file, e)
        if file.endswith("master.json"):
            emsg += "Delete it and restart rsinc to rebuild it."
        raise TypeError(emsg)
    return d 

def get_buckets(record_file):
    # datapoints = pickle.load(open(record_file, 'rb'))
    datapoints = torch.load(record_file)
    return [datapoints] 

def read(self):
        # Get the file size
        self._handle.seek(0, os.SEEK_END)
        size = self._handle.tell()

        if not size:
            # File is empty
            return None
        else:
            self._handle.seek(0)
            return json.load(self._handle) 

def get_translation_percentage(self, locale_path: str, locale: str) -> int:

        # backend stats
        po = polib.pofile(self.get_po_filename(locale_path, locale))
        not_translated = len(po.untranslated_entries())
        total = len(po.translated_entries()) + not_translated

        # frontend stats
        with open(self.get_json_filename(locale_path, locale)) as reader:
            for key, value in ujson.load(reader).items():
                total += 1
                if value == '':
                    not_translated += 1

        # mobile stats
        with open(os.path.join(locale_path, 'mobile_info.json')) as mob:
            mobile_info = ujson.load(mob)
        try:
            info = mobile_info[locale]
        except KeyError:
            if self.strict:
                raise
            info = {'total': 0, 'not_translated': 0}

        total += info['total']
        not_translated += info['not_translated']

        return (total - not_translated) * 100 // total 

def parse_file(config: Dict[str, Any], gens: Dict[str, Any], corpus_file: str) -> List[str]:

    # First, load the entire file into a dictionary,
    # then apply our custom filters to it as needed.

    paragraphs: List[str] = []

    with open(corpus_file) as infile:
        # OUR DATA: we need to separate the person talking and what they say
        paragraphs = remove_line_breaks(infile)
        paragraphs = add_flair(paragraphs, gens)

    return paragraphs 

def load_config() -> Dict[str, Any]:
    with open("zerver/tests/fixtures/config.generate_data.json") as infile:
        config = ujson.load(infile)

    return config 

def import_analytics_data(realm: Realm, import_dir: Path) -> None:
    analytics_filename = os.path.join(import_dir, "analytics.json")
    if not os.path.exists(analytics_filename):
        return

    logging.info("Importing analytics data from %s", analytics_filename)
    with open(analytics_filename) as f:
        data = ujson.load(f)

    # Process the data through the fixer functions.
    fix_datetime_fields(data, 'analytics_realmcount')
    re_map_foreign_keys(data, 'analytics_realmcount', 'realm', related_table="realm")
    update_model_ids(RealmCount, data, 'analytics_realmcount')
    bulk_import_model(data, RealmCount)

    fix_datetime_fields(data, 'analytics_usercount')
    re_map_foreign_keys(data, 'analytics_usercount', 'realm', related_table="realm")
    re_map_foreign_keys(data, 'analytics_usercount', 'user', related_table="user_profile")
    update_model_ids(UserCount, data, 'analytics_usercount')
    bulk_import_model(data, UserCount)

    fix_datetime_fields(data, 'analytics_streamcount')
    re_map_foreign_keys(data, 'analytics_streamcount', 'realm', related_table="realm")
    re_map_foreign_keys(data, 'analytics_streamcount', 'stream', related_table="stream")
    update_model_ids(StreamCount, data, 'analytics_streamcount')
    bulk_import_model(data, StreamCount) 

def get_language_translation_data(language: str) -> Dict[str, str]:
    if language == 'zh-hans':
        language = 'zh_Hans'
    elif language == 'zh-hant':
        language = 'zh_Hant'
    elif language == 'id-id':
        language = 'id_ID'
    path = os.path.join(settings.DEPLOY_ROOT, 'locale', language, 'translations.json')
    try:
        with open(path) as reader:
            return ujson.load(reader)
    except FileNotFoundError:
        print(f'Translation for {language} not found at {path}')
        return {} 

def get_language_list() -> List[Dict[str, Any]]:
    path = os.path.join(settings.DEPLOY_ROOT, 'locale', 'language_name_map.json')
    with open(path) as reader:
        languages = ujson.load(reader)
        return languages['name_map'] 

def load_config(self, config_filename):
        if not config_filename in os.listdir():
            print('cannot find ' + config_filename)
            return {}
        with open(config_filename) as f:
            return ujson.load(f)

    # loads a private RSA key from a json file 

def test_build_narrow_filter(self) -> None:
        fixtures_path = os.path.join(os.path.dirname(__file__),
                                     'fixtures/narrow.json')
        with open(fixtures_path) as f:
            scenarios = ujson.load(f)
        self.assertTrue(len(scenarios) == 9)
        for scenario in scenarios:
            narrow = scenario['narrow']
            accept_events = scenario['accept_events']
            reject_events = scenario['reject_events']
            narrow_filter = build_narrow_filter(narrow)
            for e in accept_events:
                self.assertTrue(narrow_filter(e))
            for e in reject_events:
                self.assertFalse(narrow_filter(e)) 

def read_file(self, team_output_dir: str, output_file: str) -> Any:
        full_path = os.path.join(team_output_dir, output_file)
        with open(full_path) as f:
            return ujson.load(f) 

def test_write_emoticon_data(self) -> None:
        fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
        mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
        output_dir = self.make_import_output_dir("mattermost")
        zerver_realm_emoji = write_emoticon_data(
            realm_id=3,
            custom_emoji_data=mattermost_data["emoji"],
            data_dir=self.fixture_file_name("", "mattermost_fixtures"),
            output_dir = output_dir,
        )
        self.assertEqual(len(zerver_realm_emoji), 2)
        self.assertEqual(zerver_realm_emoji[0]["file_name"], "peerdium")
        self.assertEqual(zerver_realm_emoji[0]["realm"], 3)
        self.assertEqual(zerver_realm_emoji[0]["deactivated"], False)

        self.assertEqual(zerver_realm_emoji[1]["file_name"], "tick")
        self.assertEqual(zerver_realm_emoji[1]["realm"], 3)
        self.assertEqual(zerver_realm_emoji[1]["deactivated"], False)

        records_file = os.path.join(output_dir, "emoji", "records.json")
        with open(records_file) as f:
            records_json = ujson.load(f)

        self.assertEqual(records_json[0]["file_name"], "peerdium")
        self.assertEqual(records_json[0]["realm_id"], 3)
        exported_emoji_path = self.fixture_file_name(mattermost_data["emoji"][0]["image"], "mattermost_fixtures")
        self.assertTrue(filecmp.cmp(records_json[0]["path"], exported_emoji_path))

        self.assertEqual(records_json[1]["file_name"], "tick")
        self.assertEqual(records_json[1]["realm_id"], 3)
        exported_emoji_path = self.fixture_file_name(mattermost_data["emoji"][1]["image"], "mattermost_fixtures")
        self.assertTrue(filecmp.cmp(records_json[1]["path"], exported_emoji_path)) 

def load_slack_message_conversion_tests(self) -> Dict[Any, Any]:
        test_fixtures = {}
        with open(os.path.join(os.path.dirname(__file__),
                               'fixtures/slack_message_conversion.json')) as f:
            data = ujson.load(f)
        for test in data['regular_tests']:
            test_fixtures[test['name']] = test

        return test_fixtures