Python tinydb.TinyDB() Examples

def test_serialisation_of_numpy_ndarray(tmpdir):
    from sacred.observers.tinydb_hashfs_bases import NdArraySerializer
    from tinydb_serialization import SerializationMiddleware
    import numpy as np

    # Setup Serialisation object for non list/dict objects
    serialization_store = SerializationMiddleware()
    serialization_store.register_serializer(NdArraySerializer(), "TinyArray")

    db = TinyDB(
        os.path.join(tmpdir.strpath, "metadata.json"), storage=serialization_store
    )

    eye_mat = np.eye(3)
    ones_array = np.ones(5)

    document = {"foo": "bar", "some_array": eye_mat, "nested": {"ones": ones_array}}

    db.insert(document)
    returned_doc = db.all()[0]

    assert returned_doc["foo"] == "bar"
    assert (returned_doc["some_array"] == eye_mat).all()
    assert (returned_doc["nested"]["ones"] == ones_array).all() 

def db_setup(org_name):
    """ HACK
    setup db per org as org_name.db
    setup global queries into it
    """
    db_filename = "{}.db.json".format(org_name)
    try:
        file_stat = os.stat(db_filename)
        if file_stat.st_size > 0:
            logger.warn("Updating '%s' may not work.", db_filename)
    except OSError:
        # okay if file doesn't exist
        pass
    try:
        db = tinydb.TinyDB(db_filename)
        global last_table
        last_table = db.table("GitHub")
    except Exception:
        # something very bad. provide some info
        logger.error("Can't create/read db for '{}'".format(org_name))
        raise
    return db 

def __init__(self, container: Container, **kwargs):
        if not tinydb:  # pragma: no cover
            raise IntegrationErrors.NOT_FOUND.with_params(target='tinydb')
        self._container = container
        self._path = kwargs.get('path')
        """
        Path is needed to be specified when TinyDb gets JSONStorage (default one).
        We use value of path for identifying different DBs of TinyDb.
        If the path is not specified, we can assume that the intention is to use
        MemoryStorage (if not explicitly defined) and not to cache the TinyDb instance.
        """

        self._table_name = kwargs.pop('table_name', None) or kwargs.pop('qualifier', None)
        if not self._table_name:
            raise IntegrationErrors.NO_TABLE_NAME_PROVIDED

        if self._path:
            if self._path not in self._db_cache:
                self._db_cache[self._path] = tinydb.TinyDB(**kwargs)
            self._db: tinydb.TinyDB = self._db_cache[self._path]
        else:
            if 'storage' not in kwargs:
                kwargs['storage'] = tinydb.storages.MemoryStorage
            self._db = tinydb.TinyDB(**kwargs)
        self._table: tinydb.database.Table = self._db.table(self._table_name) 

def __init__(self, name, config=None, profile=None, filecfg='~/d6tpipe/cfg.json', sortby='filename'):
        super().__init__(name, sortby)
        self.profile = 'default' if profile is None else profile
        if config is None:
            self.configmgr = ConfigManager(filecfg=filecfg, profile=self.profile)
            self.config = self.configmgr.load()
        else:
            self.config = config
            warnings.warn("Using manual config override, some api functions might not work")

        self.cfg_profile = self.config
        self._set_dir(self.name)

        # create db connection
        self._db = TinyDB(self.cfg_profile['filedb'], storage=_tdbserialization)
        self.dbfiles = self._db.table(name+'-files')
        self.dbconfig = self._db.table(name+'-cfg')

        self.settings = self.dbconfig.all()[-1]['pipe'] if self.dbconfig.all() else {}
        self.schema = self.settings.get('schema',{})

        print('Operating in local mode, use this to access local files, to run remote operations use `Pipe()`') 

def get_db_file_manager(root_dir):
    fs = HashFS(os.path.join(root_dir, "hashfs"), depth=3, width=2, algorithm="md5")

    # Setup Serialisation object for non list/dict objects
    serialization_store = SerializationMiddleware()
    serialization_store.register_serializer(DateTimeSerializer(), "TinyDate")
    serialization_store.register_serializer(FileSerializer(fs), "TinyFile")

    if opt.has_numpy:
        serialization_store.register_serializer(NdArraySerializer(), "TinyArray")
    if opt.has_pandas:
        serialization_store.register_serializer(DataFrameSerializer(), "TinyDataFrame")
        serialization_store.register_serializer(SeriesSerializer(), "TinySeries")

    db = TinyDB(os.path.join(root_dir, "metadata.json"), storage=serialization_store)
    return db, fs 

def __init__(self, db_name, domain, raise_if_table_doesnt_exist=True, only_enabled=False, only_users=False):
        self.db = None
        self.table = None
        self.only_enabled = (Query().enabled.exists() if only_enabled else Query().ntlmhash.exists()) & ( Query().enabled == True if only_enabled else Query().ntlmhash.exists())
        self.only_users = (Query().username.exists() if only_users else Query().ntlmhash.exists()) & (Query().username.test(lambda v: not v.endswith("$")) if only_users else Query().ntlmhash.exists())

        serialization = SerializationMiddleware()
        serialization.register_serializer(DateTimeSerializer(), "datetime")

        self.db = TinyDB(db_name, storage=CachingMiddleware(serialization))

        tables = list(self.db.tables())
        if raise_if_table_doesnt_exist and domain not in tables:
            raise DomainDoesntExist("Hashes for domain '{}' do not exist in database.".format(domain), tables)

        self.table = self.db.table(domain) 

def x_project_findings(rules):
    db = TinyDB("projects.json")
    project_list = []
    for project in db.table("Project").all():
        project_list.append(project['projectId'])

    for rule_title in rules:
        findings = []
        entity = {}
        for projectId in project_list:
            project_db = TinyDB("project_dbs/" + projectId + ".json")
            res = project_db.table("Rule").get(Query().title == rule_title)
            for finding in project_db.table("Finding").all():
                if finding['rule']['id'] == res.eid:
                    entity = project_db.table(finding['entity']['table']).get(doc_id = finding['entity']['id'])
                    entity['projectId'] = projectId
                    findings.append(entity)
        generate_cross_project_page("", entity.keys(), {}, findings, rule_title) 

def open_db(connection):
    """
    Open a connection to a database. The connection is a dictionary of
    information which is database-specific.
    This function looks for a TinyDB file path, with key "tinydb_path".

    :param connection: Database file path
    :type connection: dict
    :return: Database connection
    :rtype: tinydb.database.TinyDB
    :raises DatabaseError if tinydb_path is not in connection
    or if there are other problems in connecting to the database
    """
    try:
        path = connection[TINYDB_PATH]
    except Exception as exception:
        raise DatabaseError("Missing configuration error", exception)
    try:
        database = TinyDB(path)
    except Exception as exception:
        raise DatabaseError("Open connection error", exception)
    return database 

def get_latest_id(database):
    """
    Get the ID of the most recent log.

    :param database: Database connection
    :type database: tinydb.database.TinyDB
    :return: log ID or None if none
    :rtype: str or unicode
    :raises DatabaseError if there are problems in connecting to the
    database
    """
    try:
        results = database.all()
    except Exception as exception:
        raise DatabaseError("Query error", exception)
    if len(results) == 0:
        return None
    results = sorted(results, key=lambda x: x['date'])
    run = results[-1]
    return run["unique_id"] 

def get_log(database, log_id):
    """
    Get the log with the given ID as a dictionary. The log and the
    log number (the identity of this log in the database) are
    both returned.

    :param database: Database connection
    :type database: tinydb.database.TinyDB
    :param log_id: log ID
    :type log_id: str or unicode
    :return: log number and log, or None if no log exists with
    the given ID.
    :rtype: (int, dict)
    :raises DatabaseError if there are problems in connecting to the
    database
    """
    try:
        results = database.search(where('unique_id').matches(log_id))
    except Exception as exception:
        raise DatabaseError("Query error", exception)
    if len(results) > 0:
        return (results[0].eid, dict(results[0]))
    else:
        return None 

def get_filediffs(database, log_number):
    """
    Get the 'filediffs' entry for the log with the given log number.
    The log_number is returned via get_log.

    :param database: Database connection
    :type database: tinydb.database.TinyDB
    :param log_number: log number
    :type log_id: str or unicode
    :return: 'filediffs' entry or None if none
    :rtype: dict
    :raises DatabaseError if there are problems in connecting to the
    database
    """
    try:
        diffs = database.table("filediffs")
        results = diffs.search(where("run_id") == log_number)
    except Exception as exception:
        raise DatabaseError("Query error", exception)
    if len(results) > 0:
        return dict(results[0])
    else:
        return None 

def from_database(self, database: Union[str, tinydb.TinyDB], table: str = None):
        """Load history from a database supplied as a path to a file or a
        :obj:`tinydb.TinyDB` object.

        Args:
            database: :obj:`str` or :obj:`tinydb.TinyDB`. The database to load.
            table: (optional) :obj:`str`. The table to load from the database.
                This argument is not required if the database has only one table.

        Raises:
            :class:`ValueError`: if the database contains more than one table
            and `table` is not given.
        """
        super(Table, self).from_database(database, table)
        for doc in self._db_default_table:
            history_point = self._convert_doc_to_history(doc)
            self._log_history_point(history_point) 

def from_database(self, database: Union[str, tinydb.TinyDB], table: str = None):
        """Load history from a database supplied as a path to a file or a
        :obj:`tinydb.TinyDB` object.

        Args:
            database: :obj:`str` or :obj:`tinydb.TinyDB`. The database to load.
            table: (optional) :obj:`str`. The table to load from the database.
                This argument is not required if the database has only one table.

        Raises:
            :class:`ValueError`: if the database contains more than one table
            and `table` is not given.
        """
        super(Tensorboard, self).from_database(database, table)
        for doc in self._db_default_table:
            history_point = self._convert_doc_to_history(doc)
            self._log_history_point(history_point) 

def __init__(self, my_stars_home, mode):
        self._db = TinyDB(os.path.join(my_stars_home, 'mystars.db'), storage=CachingMiddleware(JSONStorage))

        if mode == 't':
            self._db.purge_tables()

        self._idx = self._db.table('index')

        if not self._idx.contains(Query().name == 'language'):
            self._idx.insert({
                'name': 'language',
                'docs': {}
            })
        if not self._idx.contains(Query().name == 'keyword'):
            self._idx.insert({
                'name': 'keyword',
                'docs': {}
            }) 

def create_ports(tmpdir, monkeypatch):
    def _create_ports(ports):
        def get_db():
            tmp_db = tmpdir.join("db.json")
            db = TinyDB(
                str(tmp_db),
                storage=CachingMiddleware(JSONStorage)
            )

            for port in ports:
                db.insert({
                    'name': port[0],
                    'port': port[1],
                    "description": port[2],
                    "protocol": port[3]
                })
            return db

        return monkeypatch.setattr(whatportis.db, "get_database", get_db)
    return  _create_ports 

def generate_db(tsv_file, db_file):
        """ Create db from tsv file """
        logger.info("Converting tsv %s to db file %s", tsv_file, db_file)
        if os.path.exists(db_file):
            os.remove(db_file)
        db = TinyDB(db_file)
        with codecs.open(tsv_file, "rb", encoding="utf-8") as f:
            row = f.readline().split("\t")
            headers = [SanskritObject(x).canonical() for x in row[0:8]]
            logger.info("Found dhatu tsv headers: {}".format(str(headers)))
            # FIXME - Rewrite from here
            for row in f:
                entries = row.split("\t")[:len(headers)]
                entries = [SanskritObject(e).canonical() for e in entries]
                j = dict(zip(headers, entries))
                db.insert(j)
        db.close()
        logger.info("Saved dhatus database") 

def number_gold_credit():
    credit = 0
    db = TinyDB(config.DATA_PATH + 'reddit_gold.json')
    data = db.all()
    db.close()

    for gold in data:

        if gold['status'] == "buy":
            # user have buy credits
            credit = credit - int(gold['quantity'])

        if gold['status'] == "refill":
            # user have buy credits
            credit = credit + int(gold['quantity'])

    return credit 

def add_address(username, address, active=True):
        # sanitize (just lower)
        username = str(unicode(username).lower())

        db = TinyDB(config.user_file)
        table = db.table(username)

        # check if address not already exist
        user_db = Query()
        data = table.count(user_db.address == address)

        if data == 0:
            table.insert({"type": "simple", "address": address, "coin": "doge", "enable": False})

            if active is True:
                UserStorage.active_user_address(username, address)
        else:
            bot_logger.logger.error("address %s already registered for  %s " % (str(address), str(username))) 

def get_user_address(cls, username):
        # sanitize (just lower)
        username = str(unicode(username).lower())

        if UserStorage.exist(username):
            db = TinyDB(config.user_file)
            table = db.table(username)
            user_db = Query()
            data = table.search(user_db.enable == True)
            if len(data) > 0:
                return data[0].get('address')
            else:
                # username not found
                return None
        else:
            bot_logger.logger.error("get address of un-registered user  %s " % (str(username))) 

def active_user_address(cls, username, address):
        # sanitize (just lower)
        username = str(unicode(username).lower())

        if UserStorage.exist(username):
            db = TinyDB(config.user_file)
            table = db.table(username)

            # check if address not already exist
            user_db = Query()
            data = table.count(user_db.address == address)

            if data == 1:
                # disable all other address
                enabled_address = table.search(user_db.enable == True)
                for item in enabled_address:
                    table.update({"enable": False}, eids=[item.eid])

                # enable only one
                table.update({"enable": True}, user_db.address == address)
            else:
                bot_logger.logger.error("active a not found address (%s)  of user  %s " % (str(address), str(username)))

        else:
            bot_logger.logger.error("active address of un-registered user  %s " % (str(username))) 

def save_resquest(self):
        if self.pattern is not None:
            db = TinyDB(config.vanitygen)
            db.insert({
                "id": self.id,
                "user": self.user.username,
                "use": self.use,
                "pattern": self.pattern,
                "finish": False,
                "address": self.address,
                "difficulty": self.difficulty,
                "duration": 0
            })
            return True
        else:
            return False 

def add_to_history(user_history, sender, receiver, amount, action, finish=False, tx_id="", tip_id=""):
        # convert object to string of name if necessary
        if type(user_history) is models.User:
            user_history = user_history.username

        if tip_id == "":
            tip_id = random.randint(0, 99999999)

        bot_logger.logger.info("Save for history user=%s, sender=%s, receiver=%s, amount=%s, action=%s, finish=%s" % (
            user_history, sender, receiver, amount, action, finish))

        db = TinyDB(config.history_path + user_history + '.json')
        db.insert({
            "id": tip_id,
            "user": user_history,
            "sender": sender,
            "receiver": receiver,
            "amount": amount,
            "action": action,
            "finish": finish,
            "status": "",
            "tx_id": tx_id,
            'time': datetime.datetime.now().isoformat(),
        })
        db.close() 

def add_to_history_tip(user_history, action, tip):
        # convert object to string of name if necessary
        if type(user_history) is models.User:
            user_history = user_history.username

        bot_logger.logger.info("Save for history user=%s, sender=%s, receiver=%s, amount=%s, action=%s, finish=%s" % (
            user_history, tip.sender.username, tip.receiver.username, tip.amount, action, tip.finish))

        db = TinyDB(config.history_path + user_history + '.json')
        db.insert({
            "user": user_history,
            "id": tip.id,
            "sender": tip.sender.username,
            "receiver": tip.receiver.username,
            "amount": tip.amount,
            "action": action,
            "finish": tip.finish,
            "status": tip.status,
            "tx_id": tip.tx_id,
            'time': tip.time,
        })
        db.close() 

def update_tip(user_history, tip):
        # convert object to string of name if necessary
        if type(user_history) is models.User:
            user_history = user_history.username

        # update only finish tips
        bot_logger.logger.info("update history for user=%s, tip.tx_id=%s" % (user_history, tip.tx_id))
        if tip.id is not None:
            bot_logger.logger.info("update history for user=%s, tip.id=%s" % (user_history, tip.id))

            db = TinyDB(config.history_path + user_history + '.json')
            tip_query = Query()
            db.update({'finish': tip.finish}, tip_query.id == tip.id)
            db.update({'tx_id': tip.tx_id}, tip_query.id == tip.id)
            db.update({'status': tip.status}, tip_query.id == tip.id)
            db.close()
        else:
            bot_logger.logger.warn("update history fail user=%s, tip.id=%s" % (user_history, tip.id)) 

def update_withdraw(user_history, status, tx_id, tip_id):
        # convert object to string of name if necessary
        if type(user_history) is models.User:
            user_history = user_history.username

        # update only finish tips
        if tip_id is not None:
            bot_logger.logger.info("update history for user=%s, tip.id=%s" % (user_history, tip_id))

            db = TinyDB(config.history_path + user_history + '.json')
            tip_query = Query()
            db.update({'finish': status}, tip_query.id == tip_id)
            db.update({'tx_id': tx_id}, tip_query.id == tip_id)
            db.update({'status': "finish"}, tip_query.id == tip_id)
            db.close()
        else:
            bot_logger.logger.warn("update history fail user=%s, tip.id=%s" % (user_history, tip_id)) 

def init_bridge():
    """Parse the configuration file and set relevant variables."""
    conf_path = os.path.abspath(os.getenv('WAT_CONF', ''))

    if not conf_path or not os.path.isfile(conf_path):
        sys.exit('Could not find configuration file')

    parser = configparser.ConfigParser()
    parser.read(conf_path)

    # Whatsapp settings
    SETTINGS['wa_phone'] = parser.get('wa', 'phone')
    SETTINGS['wa_password'] = parser.get('wa', 'password')

    # Telegram settings
    SETTINGS['owner'] = parser.getint('tg', 'owner')
    SETTINGS['tg_token'] = parser.get('tg', 'token')

    # TinyDB
    global DB
    DB = TinyDB(parser.get('db', 'path'))
    DB.table_class = SmartCacheTable 

def init_store(self):
        if self.memory_storage is True:
            return TinyDB(storage=MemoryStorage)
        if self.dbfile:
            dbfile = self.dbfile
        elif os.getenv("PAN_CREDENTIALS_DBFILE"):
            dbfile = os.getenv("PAN_CREDENTIALS_DBFILE")
        else:
            dbfile = os.path.join(
                os.path.expanduser("~"),
                ".config",
                "pan_cortex_data_lake",
                "credentials.json",
            )
        if not os.path.exists(os.path.dirname(dbfile)):
            try:
                os.makedirs(os.path.dirname(dbfile), 0o700)
            except OSError as e:
                raise CortexError("{}".format(e))
        return TinyDB(dbfile, sort_keys=True, indent=4, default_table="profiles") 

def rte_wipeall():
    tdb = TinyDB(cfg_db)
    tdb.purge_tables()
    return jsonify({'status':'wiped','db':tdb.all()}) 

def __init__(self, path, table, pk=None):
        self.table = table
        self.db = TinyDB(path, storage=_tdbserialization).table(table)
        self.pk = pk 

def save_db(self, cls_ap, path):
        with tinydb.TinyDB(path) as db:
            row = dict([(key, fn(self, cls_ap=cls_ap)) for key, fn in self.mapper.items()])
            db.insert(row)