Python bson.json_util.loads() Examples

def preparations_data(self, data):
        """
        Data preparation.

        Args:
            `data` : data from AMQP.

        Raises:
            `n6QueueProcessingException` when except processing data.
        """
        try:
            self.raw = loads(data)
            # calculate md5, inplace its fastest
            self.headers['meta'].update({
                'md5': hashlib.md5(dumps(self.raw, sort_keys=True)).hexdigest()})

        except Exception as exc:
            LOGGER.error('exception when processing: %r %r %r (%r)',
                         self.dbm.currdb, self.dbm.currcoll, data, exc)
            raise

        else:
            self.write() 

def __prepare_doc(self, policy):
        """
        Prepare Policy object as a document for insertion.
        """
        # todo - add dict inheritance
        doc = b_json.loads(policy.to_json())
        if policy.type == TYPE_STRING_BASED:
            for field in self.condition_fields:
                compiled_regexes = []
                for el in doc[field]:
                    if policy.start_tag in el and policy.end_tag in el:
                        compiled = compile_regex(el, policy.start_tag, policy.end_tag).pattern
                    else:
                        compiled = el
                    compiled_regexes.append(compiled)
                doc[self.condition_field_compiled_name(field)] = compiled_regexes
        doc['_id'] = policy.uid
        return doc 

def test_utils_montyexport(tmp_monty_repo):
    database = "dump_db_JSON"
    collection = "dump_col_JSON"

    with open_repo(tmp_monty_repo):
        montyexport(database, collection, JSON_DUMP)

        loaded_examples = list()
        loaded_exported = list()

        with open(JSON_DUMP, "r") as dump:
            data = dump.read().strip()
            for d, s in zip(data.split("\n"), SERIALIZED.split("\n")):
                loaded_exported.append(json_util.loads(d))
                loaded_examples.append(json_util.loads(s))

        sort = (lambda l: sorted(l, key=lambda i: i["_id"]))
        for d, s in zip(sort(loaded_exported), sort(loaded_examples)):
            assert d == s

        os.remove(JSON_DUMP) 

def test_binary(self):
        bin_type_dict = {"bin": Binary(b"\x00\x01\x02\x03\x04")}
        md5_type_dict = {
            "md5": Binary(b" n7\x18\xaf\t/\xd1\xd1/\x80\xca\xe7q\xcc\xac",
                          MD5_SUBTYPE)
        }
        custom_type_dict = {"custom": Binary(b"hello", USER_DEFINED_SUBTYPE)}

        self.round_trip(bin_type_dict)
        self.round_trip(md5_type_dict)
        self.round_trip(custom_type_dict)

        json_bin_dump = bsonjs_dumps(md5_type_dict)
        # Order should be $binary then $type.
        self.assertEqual(
            ('{ "md5" : { "$binary" : "IG43GK8JL9HRL4DK53HMrA==", '
             '"$type" : "05" } }'),
            json_bin_dump)

        json_bin_dump = bsonjs_dumps(custom_type_dict)
        self.assertTrue('"$type" : "80"' in json_bin_dump)
        # Check loading invalid binary
        self.assertRaises(ValueError, bsonjs.loads,
                          '{"a": {"$binary": "invalid", "$type": "80"}}') 

def update_intent(id):
    """
    Update a story from the provided json
    :return:
    """
    json_data = loads(request.get_data())
    intent = Intent.objects.get(id=ObjectId(id))
    intent = update_document(intent, json_data)
    intent.save()
    return 'success', 200 

def test_dump_throws_no_write_attribute(self):
        bson_bytes = bsonjs.loads('{ "test" : "me" }')
        not_file = {}
        self.assertRaises(AttributeError, bsonjs.dump, bson_bytes, not_file) 

def test_load_basic(self):
        json_str = '{ "test" : "me" }'
        filep = StringIO(json_str)
        self.assertEqual(bsonjs.loads(json_str), bsonjs.load(filep)) 

def test_load_unicode(self):
        json_unicode = u'{ "test" : "me" }'

        class UnicodeRead(object):
            def read(self):
                return json_unicode
        self.assertEqual(bsonjs.loads(json_unicode),
                         bsonjs.load(UnicodeRead())) 

def tweets_from_JSON_file(tweetfile):
    """
    Returns a list of tweets read from given file. Tweets are dicts representing json of 
    file contents. For a less-memory intensive version, consider the tweet_from_file_IT() 
    function.
    Note: will not check validity of tweet. Will simply return dict representations of
    JSON entities in the file
    """
    with open(tweetfile) as handle:
        tweets = loads(handle.read(), cls=ConcatJSONDecoder)
    return tweets 

def tweets_from_JSON_file_IT(tweetfile):
    """
    Returns an iterator for tweets in given tweetfile. Tweets are considered dict
    representations of JSON in given tweetfile
    """
    with open(tweetfile) as handle:
        for line in handle:
            yield loads(line.strip()) 

def _getMongoFixture(name):
        with open("tests/fixtures/mongo_responses/{}.json".format(name), "rb") as f:
            return loads(f.read()) 

def _getFixture(name):
        with open("tests/fixtures/mongo_responses/{}.json".format(name)) as f:
            return loads(f.read()) 

def test_dump_basic(self):
        json_str = '{ "test" : "me" }'
        bson_bytes = bsonjs.loads(json_str)
        filep = StringIO()
        bsonjs.dump(bson_bytes, filep)
        filep.seek(0)
        self.assertEqual(json_str, filep.read()) 

def import_json(json_file):
    json_data = json_file.read()
    # intents = Intent.objects.from_json(json_data)
    all_intents = loads(json_data)

    creates_intents = []
    for intent in all_intents:
        new_intent = Intent()
        new_intent = update_document(new_intent, intent)
        new_intent.save()
        creates_intents.append(new_intent)
    return creates_intents 

def update_entity(id):
    """
    Update a story from the provided json
    :param id:
    :return:
    """
    json_data = loads(request.get_data())
    entity = Entity.objects.get(id=ObjectId(id))
    entity = update_document(entity, json_data)
    entity.save()
    return build_response.sent_ok() 

def up(self):
        def process(doc):
            """Processor for up"""
            doc_to_save = copy.deepcopy(doc)
            rules_to_save = {}
            for name, rule_str in doc['rules'].items():
                rule = b_json.loads(rule_str)
                rule_to_save = {self._type_marker: rule['type']}
                rule_to_save.update(rule['contents'])
                rules_to_save[name] = rule_to_save
            doc_to_save['rules'] = rules_to_save
            return doc_to_save
        self._each_doc(processor=process) 

def process_result_value(self, value, dialect):
        if not value:
            return None

        # Unfortunately loads() is strict about invalid utf-8 whereas dumps()
        # is not. This can result in ValueErrors during decoding - we simply
        # log and return None for now.
        # http://bugs.python.org/issue11489
        try:
            return json_util.loads(value)
        except ValueError:
            log.error('ValueError on decoding JSON', value=value) 

def process_result_value(self, value, dialect):
        return int128_to_b36(value)


# http://bit.ly/1LbMnqu
# Can simply use this as is because though we use bson.json_util, loads()
# dumps() return standard Python dicts like the json.* equivalents
# (because these are simply called under the hood) 

def bulk_add(self, id):
        i = get_object_or_404(self.objectmanager, id=id)
        data = loads(request.data)
        nodes = []

        response = {'status': 'ok', 'message': ''}

        try:
            for node in data['nodes']:
                if node['type'] in globals() and issubclass(
                        globals()[node['type']], Observable):
                    _type = globals()[node['type']]
                    try:
                        n = _type.get_or_create(value=node['value'])
                    except ObservableValidationError as e:
                        logging.error((node, e))
                        continue

                    if node['new_tags']:
                        n.tag(node['new_tags'].split(', '))
                    nodes.append(n)

            i.add([], nodes)
        except Exception as e:
            response = {'status': 'error', 'message': str(e)}

        return render(response) 

def writeToCollectionExtend(collection, symbol, df, metadata=None):
    jsonStrings = {"_id":symbol, "symbol":symbol, "data":df.to_json(orient='records'), "metadata":metadata}
    #bsonStrings = json_util.loads(jsonStrings)
    collection.save(jsonStrings) 

def test(self, rss, page, state, result, label):
        with patch('n6.collectors.generic.CollectorWithStateMixin.__init__'), \
             patch.object(self.COLLECTOR_CLASS, 'config', self.mocked_config, create=True):
            instance = self.COLLECTOR_CLASS()
            instance._download_retry = Mock(return_value=rss)
            instance._download_retry_external = Mock(return_value=page)
            instance.load_state = Mock(return_value=state)

            if label in ('no_url', 'not_updated_page'):
                with self.assertRaises(NoNewDataException):
                    self.COLLECTOR_CLASS.get_output_data_body(instance)
            else:
                output_data_body = self.COLLECTOR_CLASS.get_output_data_body(instance)
                self.assertDictEqual(loads(output_data_body), result) 

def post(self):
        if 'file' in request.files:
            f = AttachedFile.from_upload(request.files['file'])
        else:
            data = loads(request.data)
            if 'file' in data:
                f = AttachedFile.from_upload(data['file'])
            else:
                abort(400)

        return render({'filename': url_for('api.AttachedFiles:get', id=f.id)}) 

def multidelete(self):
        data = loads(request.data)
        ids = iterify(data['ids'])
        for i, inv in enumerate(Investigation.objects(links__id__in=ids)):
            inv.modify({
                "links__id": id
            },
                       set__links__S__id="local-{}-{}".format(time.time(), i))
        self.objectmanager.objects(id__in=ids).delete()
        return render({"deleted": ids}) 

def multiupdate(self):
        data = loads(request.data)
        ids = data['ids']
        new_description = data['new']['description']
        updated = []
        for link in self.objectmanager.objects(id__in=ids):
            # link.select_related() #does not work
            # must call src and dst to dereference dbrefs and not raise an exception
            link.src
            link.dst
            link.description = new_description
            link.save()
            updated.append(link.id)

        return render({"updated": updated}) 

def multidelete(self):
        """Deletes multiple entries from the database

        :query [ObjectID] ids: Array of Element IDs
        :>json [ObjectID] deleted: Array of Element IDs that were successfully deleted
        """
        data = loads(request.data)
        ids = iterify(data['ids'])
        self.objectmanager.objects(id__in=ids).delete()
        return render({"deleted": ids}) 

def multiupdate(self):
        """Updates multiple entries from the database

        :query [ObjectID] ids: Array of Element IDs
        :query [Object] new: JSON object representing fields to update
        :>json [ObjectID] updated: Array of Element IDs that were successfully updated
        """
        data = loads(request.data)
        ids = iterify(data['ids'])
        new_data = data['new']
        self.objectmanager.objects(id__in=ids).update(new_data)
        return render({
            "updated": list(self.objectmanager.objects(ids__in=ids))
        }) 

def remove(self, id):
        i = get_object_or_404(self.objectmanager, id=id)
        data = loads(request.data)
        i.remove(iterify(data['links']), iterify(data['nodes']))

        return render(i.info()) 

def writeToCollection(collection, df, id = None):
    jsonStrings = df.to_json(orient='records')
    bsonStrings = json_util.loads(jsonStrings)
    for string in bsonStrings:
        if id is not None:
            id_string = ''.join([string[item] for item in id])
            string['_id'] = id_string
        collection.save(string) 

def bsonjs_loads(json_str):
    """Provide same API as json_util.loads"""
    return to_object(bsonjs.loads(json_str)) 

def round_trip(self, doc):
        bson_bytes = to_bson(doc)
        self.assertEqual(bson_bytes, bsonjs.loads(bsonjs.dumps(bson_bytes)))
        # Check compatibility between bsonjs and json_util
        self.assertEqual(doc, json_util.loads(
            bsonjs.dumps(bson_bytes),
            json_options=json_util.STRICT_JSON_OPTIONS))
        self.assertEqual(bson_bytes, bsonjs.loads(json_util.dumps(
            doc, json_options=json_util.STRICT_JSON_OPTIONS)))