Python ujson.loads() Examples

def getguild(self, request: web.Request):
        guild = int(request.match_info['guild'])
        req = f"""SELECT info FROM guilddata WHERE UUID = $1"""
        async with self.bot.db._conn.acquire() as connection:
            response = await connection.fetchval(req, guild)
        if response:
            data = json.loads(response)

            fdata = data
            if request.match_info['tail']:
                for item in request.match_info['tail'].split("/"):
                    if not item:
                        continue
                    try:
                        key = unquote(item)
                        if isinstance(fdata, list):
                            key = int(key)
                        fdata = fdata[key]
                    except:
                        raise web.HTTPNotFound()

            return web.json_response(fdata)
        raise web.HTTPForbidden()

    # @server.route("/", methods=["GET"]) 

def update_inst_fee(self, inst: Instrument):
        """
        更新每一个合约的手续费
        """
        try:
            fee = (await self.query('InstrumentCommissionRate', InstrumentID=inst.main_code))
            if fee:
                fee = fee[0]
                inst.fee_money = Decimal(fee['CloseRatioByMoney'])
                inst.fee_volume = Decimal(fee['CloseRatioByVolume'])
            tick = json.loads(self.redis_client.get(inst.main_code))
            inst.up_limit_ratio = (Decimal(tick['UpperLimitPrice']) -
                                   Decimal(tick['PreSettlementPrice'])) / Decimal(tick['PreSettlementPrice'])
            inst.up_limit_ratio = round(inst.up_limit_ratio, 3)
            inst.down_limit_ratio = (Decimal(tick['PreSettlementPrice']) -
                                     Decimal(tick['LowerLimitPrice'])) / Decimal(tick['PreSettlementPrice'])
            inst.down_limit_ratio = round(inst.down_limit_ratio, 3)
            inst.save(update_fields=['fee_money', 'fee_volume', 'margin_rate',
                                     'up_limit_ratio', 'down_limit_ratio'])
        except Exception as e:
            logger.error('update_inst_fee failed: %s', e, exc_info=True) 

def query_reader(ch: aioredis.Channel, cb: asyncio.Future):
        msg_list = []
        while await ch.wait_message():
            _, msg = await ch.get(encoding='utf-8')
            msg_dict = json.loads(msg)
            if 'empty' in msg_dict:
                if msg_dict['empty'] is False:
                    msg_list.append(msg_dict)
            else:
                msg_list.append(msg_dict)
            if ('bIsLast' not in msg_dict or msg_dict['bIsLast']) and not cb.done():
                cb.set_result(msg_list) 

def update_position(self):
        for _, pos in self.__shares.items():
            inst = Instrument.objects.filter(
                exchange=pos['ExchangeID'],
                product_code=re.findall('[A-Za-z]+', pos['InstrumentID'])[0]).first()
            tick = json.loads(self.redis_client.get(pos['InstrumentID']))
            if pos['Direction'] == ApiStruct.D_Buy:
                profit = Decimal(tick['LastPrice']) - Decimal(pos['OpenPrice'])
            else:
                profit = Decimal(pos['OpenPrice']) - Decimal(tick['LastPrice'])
            profit = profit * Decimal(pos['Volume']) * inst.volume_multiple
            Trade.objects.update_or_create(
                broker=self.__broker, strategy=self.__strategy, instrument=inst,
                code=pos['InstrumentID'],
                direction=DirectionType.LONG if pos['Direction'] == ApiStruct.D_Buy else DirectionType.SHORT,
                close_time__isnull=True,
                defaults={
                    'open_time': datetime.datetime.strptime(
                        pos['OpenDate']+'09', '%Y%m%d%H').replace(tzinfo=pytz.FixedOffset(480)),
                    'shares': pos['Volume'], 'filled_shares': pos['Volume'],
                    'avg_entry_price': Decimal(pos['OpenPrice']),
                    'cost': pos['Volume'] * Decimal(pos['OpenPrice']) * inst.fee_money *
                    inst.volume_multiple + pos['Volume'] * inst.fee_volume,
                    'profit': profit, 'frozen_margin': Decimal(pos['Margin'])}) 

def __init__(self, file_name, user_id):
        with open(file_name, 'r') as self.opened_file:
            self.data = self.opened_file.read()
        self.user = user_id
        self.data = ujson.loads(self.data)
        self.urls = dict()
        self.tags_dict = dict()
        self.tags_set = set()
        self.check_duplicates = dict()
        self.check_duplicates_query = Bookmark.query.filter(Bookmark.user == self.user,
                                                            Bookmark.deleted == False).all()
        for x in self.check_duplicates_query:
            self.check_duplicates[x.main_url] = x
        self.html_parser = HTMLParser.HTMLParser()
        self.valid_url = re.compile(
            r'^(?:[a-z0-9\.\-]*)://'
            r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}(?<!-)\.?)|'
            r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|'
            r'\[?[A-F0-9]*:[A-F0-9:]+\]?)'
            r'(?::\d+)?'
            r'(?:/?|[/?]\S+)$', re.IGNORECASE) 

def model(message):
    sentence = json.loads(message)
    doc = nlp(sentence)
    entities = nlp(sentence).ents
    if len(entities) == 0:
        return None

    ner = dict(
        sentence=sentence, 
        entities=[str(e) for e in entities]
    )

    cat, score = predict_sentiment(ner['sentence'])    

    output = [dict(
        entity=entity,
        sentiment=cat,
        score=score
    ) for entity in ner['entities']]

    return json.dumps(output) 

def __init__(self, file_name, user_id):
        """
        Reads data from file, loads it as JSON
        """
        with open(file_name, 'r') as self.opened_file:
            self.data = self.opened_file.read()
        self.user = user_id
        self.data = ujson.loads(self.data)
        self.urls = dict()  # Keeps track of all the urls in the import file, used when adding to db
        self.tags_dict = dict()  # Store tag objects for imported bookmarks
        self.tags_set = set()  # Keeps track of all the tags in the import file
        self.check_duplicates = dict()  # Store all current bookmarks for the user
        self.check_duplicates_query = Bookmark.query.filter(Bookmark.user == self.user,
                                                            Bookmark.deleted == False).all()
        for x in self.check_duplicates_query:
            self.check_duplicates[x.main_url] = x  # Add bookmark object to dict
        self.html_parser = HTMLParser.HTMLParser()
        self.valid_url = re.compile(
            r'^(?:[a-z0-9\.\-]*)://'
            r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}(?<!-)\.?)|'
            r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|'
            r'\[?[A-F0-9]*:[A-F0-9:]+\]?)'
            r'(?::\d+)?'
            r'(?:/?|[/?]\S+)$', re.IGNORECASE)  # We only want valid URLs in the database 

def call(self, url, params=None):
        '''Actually make the API call with the given params - this should only be called by the namespace methods - use the helpers in regular usage like m.tags.list()'''
        if params is None: params = {}
        params['key'] = self.apikey
        params = json.dumps(params)
        self.log('POST to %s%s.json: %s' % (ROOT, url, params))
        start = time.time()
        r = self.session.post('%s%s.json' % (ROOT, url), data=params, headers={'content-type': 'application/json', 'user-agent': 'Mandrill-Python/1.0.55'})
        try:
            remote_addr = r.raw._original_response.fp._sock.getpeername() # grab the remote_addr before grabbing the text since the socket will go away
        except:
            remote_addr = (None, None) #we use two private fields when getting the remote_addr, so be a little robust against errors

        response_body = r.text
        complete_time = time.time() - start
        self.log('Received %s in %.2fms: %s' % (r.status_code, complete_time * 1000, r.text))
        self.last_request = {'url': url, 'request_body': params, 'response_body': r.text, 'remote_addr': remote_addr, 'response': r, 'time': complete_time}

        result = json.loads(response_body)

        if r.status_code != requests.codes.ok:
            raise self.cast_error(result)
        return result 

def load_data(args, filename, skip_no_answer=False):
    """Load examples from preprocessed file.
    One example per line, JSON encoded.
    """
    # Load JSON lines
    with open(filename) as f:
        examples = [json.loads(line) for line in f]

    # Make case insensitive?
    if args.uncased_question or args.uncased_doc:
        for ex in examples:
            if args.uncased_question:
                ex['question'] = [w.lower() for w in ex['question']]
                ex['question_char'] = [w.lower() for w in ex['question_char']]
            if args.uncased_doc:
                ex['document'] = [w.lower() for w in ex['document']]
                ex['document_char'] = [w.lower() for w in ex['document_char']]

    # Skip unparsed (start/end) examples
    if skip_no_answer:
        examples = [ex for ex in examples if len(ex['answers']) > 0]
    return examples 

def _fetch_histograms_definition(url):
    cached = definition_cache.get(url, None)
    if cached is None:
        definition = requests.get(url).content

        # see bug 920169
        definition = definition.replace(b'"JS::gcreason::NUM_TELEMETRY_REASONS"', b"101")
        definition = definition.replace(b'"mozilla::StartupTimeline::MAX_EVENT_ID"', b"12")
        definition = definition.replace(b'"80 + 1"', b"81")

        parsed = json.loads(definition)
        parsed.update(histogram_exceptions)
        definition_cache[url] = parsed
        return parsed
    else:
        return cached 

def _thread(self, obj_key):
        article_list = []

        with NamedTemporaryFile(mode='w+b', dir='/home/ubuntu/temp2/') as packet_temp:
            s3client.download_fileobj('periodista', obj_key, packet_temp)
            packet_temp.seek(0)

            with open(packet_temp.name, 'r') as fin:
                for l in fin:
                    article = json.loads(l)

                    # Preprocessing could go here
                    _fix_notfound_authors(article)
                    _fix_photos(article)
                    article_list.append(article)
        return article_list 

def _iterate_through_archivedotorg(bucket_name):
    with NamedTemporaryFile(mode='w+b', dir='/home/ubuntu/temp2/') as packet_temp:
        s3client.download_fileobj(bucket_name, 'archivedotorg.jsonl', packet_temp)
        packet_temp.seek(0)

        with open(packet_temp.name, 'r') as fin:
            for l in fin:
                article = json.loads(l)
                article['split'] = _get_split(article['domain'])
                if article['split'] == 'ignore':
                    article['split'] = 'train'

                # Preprocessing could go here
                _fix_notfound_authors(article)
                _fix_photos(article)
                if _is_definitely_unique(article):
                    yield article 

def article_iterator(encoder, final_desired_size=1025):
    """ Iterate through the provided filename + tokenize"""
    assert os.path.exists(args.input_fn)
    with open(args.input_fn, 'r') as f:
        for l_no, l in enumerate(f):
            if l_no % args.num_folds == args.fold:
                article = json.loads(l)
                article['input_ids'] = tokenize_for_grover_training(encoder, article, desired_size=final_desired_size,
                                                                    unconditional_prob=.35)
                article['inst_index'] = (l_no // args.num_folds)
                if article['inst_index'] < 100:
                    print('---\nINPUT{}. {}\n---\nTokens: {}\n'.format(article['inst_index'],
                                                                       detokenize(encoder, article['input_ids']),
                                                                       article['input_ids']
                                                                       ), flush=True)
                if len(article['input_ids']) == 0:
                    continue
                yield article 

def test_unpack(data_dir, heka_format, try_snappy, strict, expected_count,
                expected_exception):
    count = 0
    threw_exception = False
    filename = "{}/test_{}.heka".format(data_dir, heka_format)
    with open(filename, "rb") as o:
        if "gzip" in heka_format:
            o = streaming_gzip_wrapper(o)
        try:
            for r, b in message_parser.unpack(o, try_snappy=try_snappy, strict=strict):
                j = json.loads(r.message.payload)
                assert count == j["seq"]
                count += 1
        except DecodeError:
            threw_exception = True

    assert count == expected_count
    assert threw_exception == expected_exception 

def iter_question(filename, file_map):
    with open(filename, "r") as f:
        for line in f:
            item = json.loads(line.strip())

            question = item['question']
            question_id = item['question_id']
            answer = XQAAnswer([ans.strip() for ans in item['answers']])

            docs = []
            for (subdir, filename) in item['docs']:
                title = filename[:filename.rfind('.')] if filename.rfind('.') != -1 else filename
                filepath = join(subdir, title)
                file_map[title] = filepath
                docs.append(XQADoc(title))

            yield XQAQuestion(question, question_id, answer, docs) 

def _get_bin(self, key):
        '''
        Returns a binned dictionary based on redis zscore

        @return: The sorted dict
        '''
        # keys based on score
        sortedDict = {}
        # this doesnt return them in order, need to bin first
        for item in self.redis_conn.zscan_iter(key):
            my_item = ujson.loads(item[0])
            # score is negated in redis
            my_score = -item[1]

            if my_score not in sortedDict:
                sortedDict[my_score] = []

            sortedDict[my_score].append(my_item)

        return sortedDict 

def _mini_purge(self, spiderid, appid, crawlid):
        '''
        Actually purges the crawlid from the queue

        @param spiderid: the spider id
        @param appid: the app id
        @param crawlid: the crawl id
        @return: The number of requests purged
        '''
        total_purged = 0

        match_string = '{sid}:*:queue'.format(sid=spiderid)
        # using scan for speed vs keys
        for key in self.redis_conn.scan_iter(match=match_string):
            for item in self.redis_conn.zscan_iter(key):
                item_key = item[0]
                item = ujson.loads(item_key)
                if 'meta' in item:
                    item = item['meta']

                if item['appid'] == appid and item['crawlid'] == crawlid:
                    self.redis_conn.zrem(key, item_key)
                    total_purged = total_purged + 1

        return total_purged 

def post(self):
        if not self.request.body:
            self.set_status(401)
            self.finish('error')
            return

        json = ujson.loads(self.request.body)
        print(json)

        if 'path' not in json:
            self.set_status(401)
            self.finish('error')
            return

        path = json.pop('path')
        if not path or not json.get('title', None) or not json.get('authors', None) or not json.get('tags', None) or not json.get('tldr', None):
            self.set_status(401)
            self.finish('error')
            return

        kp = nb_to_kp(path, **json)
        print(kp)
        self.finish('test') 

def _listen(self):
        pubsub = self.SR.pubsub(ignore_subscribe_messages=True)
        pubsub.psubscribe('mm-engine-status.*')

        while pubsub.subscribed:
            response = pubsub.get_message(timeout=30.0)
            if response is None:
                continue

            if not bool(self._signal.receivers):
                LOG.info('no receivers')
                continue

            data = json.loads(response['data'])
            source = data.pop('source', '<unknown-node>')

            self._signal.send(source, data=data) 

def _amqp_callback(self, msg):
        try:
            LOG.info(u'{}'.format(msg.body))
            message = ujson.loads(msg.body)
            self._msg_queue.put(message)
            self._do_process.set()

        except gevent.GreenletExit:
            raise

        except:
            LOG.exception(
                "%s - exception handling syslog message",
                self.name
            ) 

def _amqp_callback(self, msg):
        try:
            message = ujson.loads(msg.body)
            self._handle_syslog_message(message)

        except gevent.GreenletExit:
            raise

        except:
            LOG.exception(
                "%s - exception handling syslog message",
                self.name
            ) 

def add_to_store(self, key, val):
        try:
            existing = self.store.get(key)
            if isinstance(existing, (bytes, bytearray)):
                existing = existing.decode()
            existing = json.loads(existing)
        except KeyError:
            existing = []
        self.store.put(key, json.dumps([*existing, val])) 

def get_custom_metadata(self):
        cmetadata = self._get(CUSTOM_METADATA)
        if cmetadata is None:
            return None
        return ujson.loads(cmetadata) 

def _process_item(self, row):
        row.pop(None, None)  # I love this

        result = {}

        indicator = row.get('Name', '')
        if indicator == '':
            return []

        risk = row.get('Risk', '')
        if risk != '':
            try:
                result['recordedfuture_risk'] = int(risk)
                result['confidence'] = (int(risk) * self.confidence) / 100
            except:
                LOG.debug("%s - invalid risk string: %s",
                          self.name, risk)

        riskstring = row.get('RiskString', '')
        if riskstring != '':
            result['recordedfuture_riskstring'] = riskstring

        edetails = row.get('EvidenceDetails', '')
        if edetails != '':
            try:
                edetails = ujson.loads(edetails)
            except:
                LOG.debug("%s - invalid JSON string in EvidenceDetails: %s",
                          self.name, edetails)
            else:
                edetails = edetails.get('EvidenceDetails', [])
                result['recordedfuture_evidencedetails'] = \
                    [ed['Rule'] for ed in edetails]

        result['recordedfuture_entityurl'] = \
            'https://app.recordedfuture.com/live/sc/entity/idn:' + indicator

        return [[indicator, result]] 

def get(self, key):
        if type(key) == unicode:
            key = key.encode('utf8')

        ikey = self._indicator_key(key)
        value = self._get(ikey)
        if value is None:
            return None

        # skip version
        return ujson.loads(value[8:]) 

def _process_item(self, item):
        indicator = item[0]
        value = json.loads(item[2])
        value['type'] = item[1]
        value['_expiration_ts'] = item[3]

        if value['_expiration_ts'] is None:
            # if none, expiration is set to update_ts+default_ttl
            value['_expiration_ts'] = item[4]+self.default_ttl*1000

        return [[indicator, value]] 

def _callback(self, msg):
        try:
            msg = json.loads(msg)
        except ValueError:
            LOG.error("invalid message received")
            return

        method = msg.get('method', None)
        params = msg.get('params', {})
        if method is None:
            LOG.error("Message without method field")
            return

        if method not in self.allowed_methods:
            LOG.error("Method not allowed: %s", method)
            return

        m = getattr(self.object, method, None)
        if m is None:
            LOG.error('Method %s not defined', method)
            return

        try:
            m(**params)

        except gevent.GreenletExit:
            raise

        except:
            LOG.exception('Exception in handling %s on topic %s '
                          'with params %s', method, self.topic, params)

        self.num_callbacks += 1 

def get_parsed(msg, only_incoming=None, only_outgoing=None):
        assert not (only_incoming and only_outgoing)
        if isinstance(msg, (bytes, bytearray)):
            msg = msg.decode()
        msg = json.loads(msg)
        if only_incoming:
            return Recorder.filter_incoming(msg)
        if only_outgoing:
            return Recorder.filter_outgoing(msg)
        return msg 

def deserializeMsg(msg):
        if isinstance(msg, bytes):
            msg = msg.decode()
        msg = json.loads(msg)
        return msg 

def add_start_time(data_directory, tm):
    if not os.path.isdir(data_directory):
        os.makedirs(data_directory)
    file_name = os.path.join(data_directory, 'start_times')
    if not os.path.isfile(file_name):
        start_times = []
    else:
        with open(file_name, 'r') as f:
            start_times = json.loads(f.read())
            if len(start_times[-1]) != 2:
                raise RuntimeError('Wrongly formatted start_times file')
    start_times.append([tm, ])
    with open(file_name, 'w+') as f:
        f.write(json.dumps(start_times))