Python sqlalchemy.exists() Examples

The following are 30 code examples for showing how to use sqlalchemy.exists(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module sqlalchemy , or try the search function .

Example 1
Project: rasa-for-botfront   Author: botfront   File: tracker_store.py    License: Apache License 2.0 7 votes vote down vote up
def get_or_create_table(
        self, table_name: Text
    ) -> "boto3.resources.factory.dynamodb.Table":
        """Returns table or creates one if the table name is not in the table list"""
        import boto3

        dynamo = boto3.resource("dynamodb", region_name=self.region)
        if self.table_name not in self.client.list_tables()["TableNames"]:
            table = dynamo.create_table(
                TableName=self.table_name,
                KeySchema=[
                    {"AttributeName": "sender_id", "KeyType": "HASH"},
                    {"AttributeName": "session_date", "KeyType": "RANGE"},
                ],
                AttributeDefinitions=[
                    {"AttributeName": "sender_id", "AttributeType": "S"},
                    {"AttributeName": "session_date", "AttributeType": "N"},
                ],
                ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5},
            )

            # Wait until the table exists.
            table.meta.client.get_waiter("table_exists").wait(TableName=table_name)
        return dynamo.Table(table_name) 
Example 2
Project: hydrus   Author: HTTP-APIs   File: crud.py    License: MIT License 6 votes vote down vote up
def insert_single(object_: Dict[str, Any], session: scoped_session) -> Any:
    """Insert instance of classes with single objects.
    :param object_: object to be inserted
    :param session: sqlalchemy scoped session
    :return:

    Raises:
        ClassNotFound: If `type_` does not represt a valid/defined RDFClass.
        Instance: If an Instance of type `type_` already exists.

    """
    try:
        rdf_class = session.query(RDFClass).filter(
            RDFClass.name == object_["@type"]).one()
    except NoResultFound:
        raise ClassNotFound(type_=object_["@type"])

    try:
        session.query(Instance).filter(
            Instance.type_ == rdf_class.id).all()[-1]
    except (NoResultFound, IndexError, ValueError):
        return insert(object_, session=session)

    raise InstanceExists(type_=rdf_class.name) 
Example 3
Project: calibre-web   Author: janeczku   File: ub.py    License: GNU General Public License v3.0 6 votes vote down vote up
def init_db(app_db_path):
    # Open session for database connection
    global session
    global app_DB_path

    app_DB_path = app_db_path
    engine = create_engine(u'sqlite:///{0}'.format(app_db_path), echo=False)

    Session = sessionmaker()
    Session.configure(bind=engine)
    session = Session()

    if os.path.exists(app_db_path):
        Base.metadata.create_all(engine)
        migrate_Database(session)
        clean_database(session)
    else:
        Base.metadata.create_all(engine)
        create_admin_user(session)
        create_anonymous_user(session) 
Example 4
Project: rucio   Author: rucio   File: replica.py    License: Apache License 2.0 6 votes vote down vote up
def set_tombstone(rse_id, scope, name, tombstone=OBSOLETE, session=None):
    """
    Sets a tombstone on a replica.

    :param rse_id: ID of RSE.
    :param scope: scope of the replica DID.
    :param name: name of the replica DID.
    :param tombstone: the tombstone to set. Default is OBSOLETE
    :param session: database session in use.
    """
    stmt = update(models.RSEFileAssociation).where(and_(models.RSEFileAssociation.rse_id == rse_id, models.RSEFileAssociation.name == name, models.RSEFileAssociation.scope == scope,
                                                        ~session.query(models.ReplicaLock).filter_by(scope=scope, name=name, rse_id=rse_id).exists()))\
                                            .values(tombstone=tombstone)
    result = session.execute(stmt)
    if not result.rowcount:
        try:
            session.query(models.RSEFileAssociation).filter_by(scope=scope, name=name, rse_id=rse_id).one()
            raise exception.ReplicaIsLocked('Replica %s:%s on RSE %s is locked.' % (scope, name, get_rse_name(rse_id=rse_id, session=session)))
        except NoResultFound:
            raise exception.ReplicaNotFound('Replica %s:%s on RSE %s could not be found.' % (scope, name, get_rse_name(rse_id=rse_id, session=session))) 
Example 5
Project: daf-recipes   Author: italia   File: ckanharvester.py    License: GNU General Public License v3.0 6 votes vote down vote up
def _last_error_free_job(cls, harvest_job):
        # TODO weed out cancelled jobs somehow.
        # look for jobs with no gather errors
        jobs = \
            model.Session.query(HarvestJob) \
                 .filter(HarvestJob.source == harvest_job.source) \
                 .filter(HarvestJob.gather_started != None) \
                 .filter(HarvestJob.status == 'Finished') \
                 .filter(HarvestJob.id != harvest_job.id) \
                 .filter(
                     ~exists().where(
                         HarvestGatherError.harvest_job_id == HarvestJob.id)) \
                 .order_by(HarvestJob.gather_started.desc())
        # now check them until we find one with no fetch/import errors
        # (looping rather than doing sql, in case there are lots of objects
        # and lots of jobs)
        for job in jobs:
            for obj in job.objects:
                if obj.current is False and \
                        obj.report_status != 'not modified':
                    # unsuccessful, so go onto the next job
                    break
            else:
                return job 
Example 6
Project: mini-key-server   Author: usrbinsam   File: keymanager.py    License: MIT License 6 votes vote down vote up
def key_valid_const(app_id: int, token: str, origin: Origin) -> bool:
    """Constant time check to see if `token` exists in the database. Compares
    against all keys even if a match is found. Validates against the app id
    and the hardware id provided."""
    current_app.logger.info(f"key lookup by token {token} from {origin}")
    found = False
    for key in Key.query.all():
        if (compare_digest(token, key.token) and
                key.enabled and key.app_id == app_id
                and compare_digest(origin.hwid, key.hwid)):

            found = True
            key.last_check_ts = datetime.utcnow()
            key.last_check_ip = origin.ip
            key.total_checks += 1
            AuditLog.from_key(key, f"key check from {origin}", Event.KeyAccess)
    return found 
Example 7
Project: tacker   Author: openstack   File: vnfm_db.py    License: Apache License 2.0 6 votes vote down vote up
def choose_vnfd(self, context, service_type,
                    required_attributes=None):
        required_attributes = required_attributes or []
        LOG.debug('required_attributes %s', required_attributes)
        with context.session.begin(subtransactions=True):
            query = (
                context.session.query(VNFD).
                filter(
                    sa.exists().
                    where(sa.and_(
                        VNFD.id == ServiceType.vnfd_id,
                        ServiceType.service_type == service_type))))
            for key in required_attributes:
                query = query.filter(
                    sa.exists().
                    where(sa.and_(
                        VNFD.id ==
                        VNFDAttribute.vnfd_id,
                        VNFDAttribute.key == key)))
            LOG.debug('statements %s', query)
            vnfd_db = query.first()
            if vnfd_db:
                return self._make_vnfd_dict(vnfd_db) 
Example 8
Project: rasa-for-botfront   Author: botfront   File: tracker_store.py    License: Apache License 2.0 6 votes vote down vote up
def ensure_schema_exists(session: "Session") -> None:
    """Ensure that the requested PostgreSQL schema exists in the database.

    Args:
        session: Session used to inspect the database.

    Raises:
        `ValueError` if the requested schema does not exist.
    """
    schema_name = os.environ.get(POSTGRESQL_SCHEMA)

    if not schema_name:
        return

    engine = session.get_bind()

    if is_postgresql_url(engine.url):
        query = sa.exists(
            sa.select([(sa.text("schema_name"))])
            .select_from(sa.text("information_schema.schemata"))
            .where(sa.text(f"schema_name = '{schema_name}'"))
        )
        if not session.query(query).scalar():
            raise ValueError(schema_name) 
Example 9
Project: rasa-for-botfront   Author: botfront   File: tracker_store.py    License: Apache License 2.0 6 votes vote down vote up
def _create_database(engine: "Engine", db: Text):
        """Create database `db` on `engine` if it does not exist."""

        import psycopg2

        conn = engine.connect()

        cursor = conn.connection.cursor()
        cursor.execute("COMMIT")
        cursor.execute(f"SELECT 1 FROM pg_catalog.pg_database WHERE datname = '{db}'")
        exists = cursor.fetchone()
        if not exists:
            try:
                cursor.execute(f"CREATE DATABASE {db}")
            except psycopg2.IntegrityError as e:
                logger.error(f"Could not create database '{db}': {e}")

        cursor.close()
        conn.close() 
Example 10
Project: forseti-security   Author: forseti-security   File: storage.py    License: Apache License 2.0 6 votes vote down vote up
def type_exists(cls,
                    session,
                    inventory_index_id,
                    type_list=None):
        """Check if certain types of resources exists in the inventory.

        Args:
            session (object): Database session.
            inventory_index_id (str): the id of the inventory to query.
            type_list (list): List of types to check.

        Returns:
            bool: If these types of resources exists.
        """
        return session.query(exists().where(and_(
            Inventory.inventory_index_id == inventory_index_id,
            Inventory.category == Categories.resource,
            Inventory.resource_type.in_(type_list)
        ))).scalar() 
Example 11
Project: sqlalchemy   Author: sqlalchemy   File: test_update.py    License: MIT License 6 votes vote down vote up
def test_correlation_to_extra(self):
        users, addresses = self.tables.users, self.tables.addresses

        stmt = (
            users.update()
            .values(name="newname")
            .where(users.c.id == addresses.c.user_id)
            .where(
                ~exists()
                .where(addresses.c.user_id == users.c.id)
                .where(addresses.c.email_address == "foo")
                .correlate(addresses)
            )
        )

        self.assert_compile(
            stmt,
            "UPDATE users SET name=:name FROM addresses WHERE "
            "users.id = addresses.user_id AND NOT "
            "(EXISTS (SELECT * FROM users WHERE addresses.user_id = users.id "
            "AND addresses.email_address = :email_address_1))",
        ) 
Example 12
Project: sqlalchemy   Author: sqlalchemy   File: test_update.py    License: MIT License 6 votes vote down vote up
def test_dont_correlate_to_extra(self):
        users, addresses = self.tables.users, self.tables.addresses

        stmt = (
            users.update()
            .values(name="newname")
            .where(users.c.id == addresses.c.user_id)
            .where(
                ~exists()
                .where(addresses.c.user_id == users.c.id)
                .where(addresses.c.email_address == "foo")
                .correlate()
            )
        )

        self.assert_compile(
            stmt,
            "UPDATE users SET name=:name FROM addresses WHERE "
            "users.id = addresses.user_id AND NOT "
            "(EXISTS (SELECT * FROM addresses, users "
            "WHERE addresses.user_id = users.id "
            "AND addresses.email_address = :email_address_1))",
        ) 
Example 13
Project: sqlalchemy   Author: sqlalchemy   File: test_update.py    License: MIT License 6 votes vote down vote up
def test_autocorrelate_error(self):
        users, addresses = self.tables.users, self.tables.addresses

        stmt = (
            users.update()
            .values(name="newname")
            .where(users.c.id == addresses.c.user_id)
            .where(
                ~exists()
                .where(addresses.c.user_id == users.c.id)
                .where(addresses.c.email_address == "foo")
            )
        )

        assert_raises_message(
            exc.InvalidRequestError,
            ".*returned no FROM clauses due to auto-correlation.*",
            stmt.compile,
            dialect=default.StrCompileDialect(),
        ) 
Example 14
Project: sqlalchemy   Author: sqlalchemy   File: test_delete.py    License: MIT License 6 votes vote down vote up
def test_correlation_to_extra(self):
        table1, table2 = self.tables.mytable, self.tables.myothertable

        stmt = (
            table1.delete()
            .where(table1.c.myid == table2.c.otherid)
            .where(
                ~exists()
                .where(table2.c.otherid == table1.c.myid)
                .where(table2.c.othername == "x")
                .correlate(table2)
            )
        )

        self.assert_compile(
            stmt,
            "DELETE FROM mytable , myothertable WHERE mytable.myid = "
            "myothertable.otherid AND NOT (EXISTS "
            "(SELECT * FROM mytable WHERE myothertable.otherid = "
            "mytable.myid AND myothertable.othername = :othername_1))",
        ) 
Example 15
Project: sqlalchemy   Author: sqlalchemy   File: test_delete.py    License: MIT License 6 votes vote down vote up
def test_dont_correlate_to_extra(self):
        table1, table2 = self.tables.mytable, self.tables.myothertable

        stmt = (
            table1.delete()
            .where(table1.c.myid == table2.c.otherid)
            .where(
                ~exists()
                .where(table2.c.otherid == table1.c.myid)
                .where(table2.c.othername == "x")
                .correlate()
            )
        )

        self.assert_compile(
            stmt,
            "DELETE FROM mytable , myothertable WHERE mytable.myid = "
            "myothertable.otherid AND NOT (EXISTS "
            "(SELECT * FROM myothertable, mytable "
            "WHERE myothertable.otherid = "
            "mytable.myid AND myothertable.othername = :othername_1))",
        ) 
Example 16
Project: sqlalchemy   Author: sqlalchemy   File: test_delete.py    License: MIT License 6 votes vote down vote up
def test_autocorrelate_error(self):
        table1, table2 = self.tables.mytable, self.tables.myothertable

        stmt = (
            table1.delete()
            .where(table1.c.myid == table2.c.otherid)
            .where(
                ~exists()
                .where(table2.c.otherid == table1.c.myid)
                .where(table2.c.othername == "x")
            )
        )

        assert_raises_message(
            exc.InvalidRequestError,
            ".*returned no FROM clauses due to auto-correlation.*",
            stmt.compile,
            dialect=default.StrCompileDialect(),
        ) 
Example 17
Project: sqlalchemy   Author: sqlalchemy   File: test_froms.py    License: MIT License 6 votes vote down vote up
def test_correlate_to_union_newstyle(self):
        User = self.classes.User

        q = future_select(User).apply_labels()

        q = future_select(User).union(q).apply_labels().subquery()

        u_alias = aliased(User)

        raw_subq = exists().where(u_alias.id > q.c[0])

        self.assert_compile(
            future_select(q, raw_subq).apply_labels(),
            "SELECT anon_1.users_id AS anon_1_users_id, "
            "anon_1.users_name AS anon_1_users_name, "
            "EXISTS (SELECT * FROM users AS users_1 "
            "WHERE users_1.id > anon_1.users_id) AS anon_2 "
            "FROM ("
            "SELECT users.id AS users_id, users.name AS users_name FROM users "
            "UNION SELECT users.id AS users_id, users.name AS users_name "
            "FROM users) AS anon_1",
        ) 
Example 18
Project: sqlalchemy   Author: sqlalchemy   File: test_query.py    License: MIT License 6 votes vote down vote up
def test_exists(self):
        User = self.classes.User
        sess = create_session()

        q1 = sess.query(User)
        self.assert_compile(
            sess.query(q1.exists()),
            "SELECT EXISTS (" "SELECT 1 FROM users" ") AS anon_1",
        )

        q2 = sess.query(User).filter(User.name == "fred")
        self.assert_compile(
            sess.query(q2.exists()),
            "SELECT EXISTS ("
            "SELECT 1 FROM users WHERE users.name = :name_1"
            ") AS anon_1",
        ) 
Example 19
Project: hydrus   Author: HTTP-APIs   File: user.py    License: MIT License 5 votes vote down vote up
def add_user(id_: int, paraphrase: str, session: Session) -> None:
    """Add new users to the database.

    Raises:
        UserExits: If a user with `id_` already exists.

    """
    if session.query(exists().where(User.id == id_)).scalar():
        raise UserExists(id_=id_)
    else:
        new_user = User(id=id_, paraphrase=sha224(
            paraphrase.encode('utf-8')).hexdigest())
        session.add(new_user)
        session.commit() 
Example 20
Project: hydrus   Author: HTTP-APIs   File: crud.py    License: MIT License 5 votes vote down vote up
def get_single(type_: str, api_name: str, session: scoped_session,
               path: str = None) -> Dict[str, Any]:
    """Get instance of classes with single objects.
    :param type_: type of object to be updated
    :param api_name: api name specified while starting server
    :param session: sqlalchemy scoped session
    :param path: endpoint
    :return: response containing information about a single object

    Raises:
        ClassNotFound: If `type_` does not represt a valid/defined RDFClass.
        InstanceNotFound: If no Instance with type `type_` exists.

    """
    try:
        rdf_class = session.query(RDFClass).filter(
            RDFClass.name == type_).one()
    except NoResultFound:
        raise ClassNotFound(type_=type_)

    try:
        instance = session.query(Instance).filter(
            Instance.type_ == rdf_class.id).all()[-1]
    except (NoResultFound, IndexError, ValueError):
        raise InstanceNotFound(type_=rdf_class.name)
    object_ = get(instance.id, rdf_class.name,
                  session=session, api_name=api_name, path=path)
    if path is not None:
        object_["@id"] = "/{}/{}".format(api_name, path)
    else:
        object_["@id"] = "/{}/{}".format(api_name, type_)
    return object_ 
Example 21
Project: hydrus   Author: HTTP-APIs   File: crud.py    License: MIT License 5 votes vote down vote up
def update_single(object_: Dict[str,
                                Any],
                  session: scoped_session,
                  api_name: str,
                  link_props: Dict[str, Any],
                  path: str = None) -> int:
    """Update instance of classes with single objects.
    :param object_: new object
    :param session: sqlalchemy scoped session
    :param api_name: api name specified while starting server
    :param link_props: Link properties of the object being updated
    :param path: endpoint
    :return: id of the updated object

    Raises:
        ClassNotFound: If `object['@type']` does not represt a valid/defined RDFClass.
        InstanceNotFound: If no Instance of the class exists.

    """
    try:
        rdf_class = session.query(RDFClass).filter(
            RDFClass.name == object_["@type"]).one()
    except NoResultFound:
        raise ClassNotFound(type_=object_["@type"])

    try:
        instance = session.query(Instance).filter(
            Instance.type_ == rdf_class.id).all()[-1]
    except (NoResultFound, IndexError, ValueError):
        raise InstanceNotFound(type_=rdf_class.name)

    return update(
        id_=instance.id,
        type_=object_["@type"],
        object_=object_,
        session=session,
        api_name=api_name,
        link_props=link_props,
        path=path) 
Example 22
Project: hydrus   Author: HTTP-APIs   File: doc_parse.py    License: MIT License 5 votes vote down vote up
def insert_classes(classes: List[Dict[str, Any]],
                   session: scoped_session) -> Optional[Any]:
    """Insert all the classes as defined in the APIDocumentation into DB.

    Raises:
        TypeError: If `session` is not an instance of `scoped_session` or `Session`.

    """
    # print(session.query(exists().where(RDFClass.name == "Datastream")).scalar())
    if not isinstance(session, scoped_session) and not isinstance(
            session, Session):
        raise TypeError(
            "session is not of type <sqlalchemy.orm.scoping.scoped_session>"
            "or <sqlalchemy.orm.session.Session>"
        )
    class_list = [RDFClass(name=class_["label"].strip('.')) for class_ in classes
                  if "label" in class_ and
                  not session.query(exists().where(RDFClass.name == class_["label"]
                                                   .strip('.'))).scalar()]

    class_list.extend([RDFClass(name=class_["title"].strip('.')) for class_ in classes
                       if "title" in class_ and
                       not session.query(exists().where(RDFClass.name == class_["title"]
                                                                .strip('.'))).scalar()])
    # print(class_list)
    session.add_all(class_list)
    session.commit()
    return None 
Example 23
Project: hydrus   Author: HTTP-APIs   File: doc_parse.py    License: MIT License 5 votes vote down vote up
def insert_properties(properties: Set[str],
                      session: scoped_session) -> Optional[Any]:
    """Insert all the properties as defined in the APIDocumentation into DB."""
    prop_list = [BaseProperty(name=prop) for prop in properties
                 if not session.query(exists().where(BaseProperty.name == prop)).scalar()]
    session.add_all(prop_list)
    session.commit()
    return None


# if __name__ == "__main__":
#     Session = sessionmaker(bind=engine)
#     session = Session()
#
#     doc = doc_gen("test", "test")
#     # Extract all classes with supportedProperty from both
#     classes = get_classes(doc.generate())
#
#     # Extract all properties from both
#     # import pdb; pdb.set_trace()
#     properties = get_all_properties(classes)
#     # Add all the classes
#     insert_classes(classes, session)
#     print("Classes inserted successfully")
#     # Add all the properties
#     insert_properties(properties, session)
#     print("Properties inserted successfully") 
Example 24
Project: rucio   Author: rucio   File: replica.py    License: Apache License 2.0 5 votes vote down vote up
def __exists_replicas(rse_id, scope=None, name=None, path=None, session=None):
    """
    Internal method to check if a replica exists at a given site.
    :param rse_id: The RSE id.
    :param scope: The scope of the file.
    :param name: The name of the file.
    :param path: The path of the replica.
    :param session: The database session in use.
    """

    already_declared = False
    if path:
        path_clause = [models.RSEFileAssociation.path == path]
        if path.startswith('/'):
            path_clause.append(models.RSEFileAssociation.path == path[1:])
        else:
            path_clause.append(models.RSEFileAssociation.path == '/%s' % path)
        query = session.query(models.RSEFileAssociation.path, models.RSEFileAssociation.scope, models.RSEFileAssociation.name, models.RSEFileAssociation.rse_id, models.RSEFileAssociation.bytes).\
            with_hint(models.RSEFileAssociation, "+ index(replicas REPLICAS_PATH_IDX", 'oracle').\
            filter(models.RSEFileAssociation.rse_id == rse_id).filter(or_(*path_clause))
    else:
        query = session.query(models.RSEFileAssociation.path, models.RSEFileAssociation.scope, models.RSEFileAssociation.name, models.RSEFileAssociation.rse_id, models.RSEFileAssociation.bytes).\
            filter_by(rse_id=rse_id, scope=scope, name=name)
    if query.count():
        result = query.first()
        path, scope, name, rse_id, size = result
        # Now we check that the replica is not already declared bad
        query = session.query(models.BadReplicas.scope, models.BadReplicas.name, models.BadReplicas.rse_id, models.BadReplicas.state).\
            filter_by(rse_id=rse_id, scope=scope, name=name, state=BadFilesStatus.BAD)
        if query.count():
            already_declared = True
        return True, scope, name, already_declared, size
    else:
        return False, None, None, already_declared, None 
Example 25
Project: rucio   Author: rucio   File: replica.py    License: Apache License 2.0 5 votes vote down vote up
def __bulk_add_new_file_dids(files, account, dataset_meta=None, session=None):
    """
    Bulk add new dids.

    :param dids: the list of new files.
    :param account: The account owner.
    :param session: The database session in use.
    :returns: True is successful.
    """
    for file in files:
        new_did = models.DataIdentifier(scope=file['scope'], name=file['name'],
                                        account=file.get('account') or account,
                                        did_type=DIDType.FILE, bytes=file['bytes'],
                                        md5=file.get('md5'), adler32=file.get('adler32'),
                                        is_new=None)
        for key in file.get('meta', []):
            new_did.update({key: file['meta'][key]})
        for key in dataset_meta or {}:
            new_did.update({key: dataset_meta[key]})

        new_did.save(session=session, flush=False)
    try:
        session.flush()
    except IntegrityError as error:
        raise exception.RucioException(error.args)
    except DatabaseError as error:
        raise exception.RucioException(error.args)
    except FlushError as error:
        if match('New instance .* with identity key .* conflicts with persistent instance', error.args[0]):
            raise exception.DataIdentifierAlreadyExists('Data Identifier already exists!')
        raise exception.RucioException(error.args)
    return True 
Example 26
Project: rucio   Author: rucio   File: replica.py    License: Apache License 2.0 5 votes vote down vote up
def get_cleaned_updated_collection_replicas(total_workers, worker_number, session=None):
    """
    Get update request for collection replicas.
    :param total_workers:      Number of total workers.
    :param worker_number:      id of the executing worker.
    :param session:            Database session in use.
    :returns:                  List of update requests for collection replicas.
    """
    # Delete duplicates
    replica_update_requests = session.query(models.UpdatedCollectionReplica)
    update_requests_with_rse_id = []
    update_requests_without_rse_id = []
    duplicate_request_ids = []
    for update_request in replica_update_requests.all():
        if update_request.rse_id is not None:
            small_request = {'name': update_request.name, 'scope': update_request.scope, 'rse_id': update_request.rse_id}
            if small_request not in update_requests_with_rse_id:
                update_requests_with_rse_id.append(small_request)
            else:
                duplicate_request_ids.append(update_request.id)
                continue
        else:
            small_request = {'name': update_request.name, 'scope': update_request.scope}
            if small_request not in update_requests_without_rse_id:
                update_requests_without_rse_id.append(small_request)
            else:
                duplicate_request_ids.append(update_request.id)
                continue
    replica_update_requests.filter(models.UpdatedCollectionReplica.id.in_(duplicate_request_ids)).delete(synchronize_session=False)

    # Delete update requests which do not have collection_replicas
    session.query(models.UpdatedCollectionReplica).filter(models.UpdatedCollectionReplica.rse_id.is_(None)
                                                          & ~exists().where(and_(models.CollectionReplica.name == models.UpdatedCollectionReplica.name,  # NOQA: W503
                                                                                 models.CollectionReplica.scope == models.UpdatedCollectionReplica.scope))).delete(synchronize_session=False)
    session.query(models.UpdatedCollectionReplica).filter(models.UpdatedCollectionReplica.rse_id.isnot(None)
                                                          & ~exists().where(and_(models.CollectionReplica.name == models.UpdatedCollectionReplica.name,  # NOQA: W503
                                                                                 models.CollectionReplica.scope == models.UpdatedCollectionReplica.scope,
                                                                                 models.CollectionReplica.rse_id == models.UpdatedCollectionReplica.rse_id))).delete(synchronize_session=False)

    query = session.query(models.UpdatedCollectionReplica)
    return [update_request.to_dict() for update_request in query.all()] 
Example 27
Project: rucio   Author: rucio   File: replica.py    License: Apache License 2.0 5 votes vote down vote up
def bulk_add_bad_replicas(replicas, account, state=BadFilesStatus.TEMPORARY_UNAVAILABLE, reason=None, expires_at=None, session=None):
    """
    Bulk add new bad replicas.

    :param replicas: the list of bad replicas.
    :param account: The account who declared the bad replicas.
    :param state: The state of the file (SUSPICIOUS, BAD or TEMPORARY_UNAVAILABLE).
    :param session: The database session in use.

    :returns: True is successful.
    """
    for replica in replicas:
        insert_new_row = True
        if state == BadFilesStatus.TEMPORARY_UNAVAILABLE:
            query = session.query(models.BadReplicas).filter_by(scope=replica['scope'], name=replica['name'], rse_id=replica['rse_id'], state=state)
            if query.count():
                query.update({'state': BadFilesStatus.TEMPORARY_UNAVAILABLE, 'updated_at': datetime.utcnow(), 'account': account, 'reason': reason, 'expires_at': expires_at}, synchronize_session=False)
                insert_new_row = False
        if insert_new_row:
            new_bad_replica = models.BadReplicas(scope=replica['scope'], name=replica['name'], rse_id=replica['rse_id'], reason=reason,
                                                 state=state, account=account, bytes=None, expires_at=expires_at)
            new_bad_replica.save(session=session, flush=False)
    try:
        session.flush()
    except IntegrityError as error:
        raise exception.RucioException(error.args)
    except DatabaseError as error:
        raise exception.RucioException(error.args)
    except FlushError as error:
        if match('New instance .* with identity key .* conflicts with persistent instance', error.args[0]):
            raise exception.DataIdentifierAlreadyExists('Data Identifier already exists!')
        raise exception.RucioException(error.args)
    return True 
Example 28
Project: rucio   Author: rucio   File: replica.py    License: Apache License 2.0 5 votes vote down vote up
def add_bad_pfns(pfns, account, state, reason=None, expires_at=None, session=None):
    """
    Add bad PFNs.

    :param pfns: the list of new files.
    :param account: The account who declared the bad replicas.
    :param state: One of the possible states : BAD, SUSPICIOUS, TEMPORARY_UNAVAILABLE.
    :param reason: A string describing the reason of the loss.
    :param expires_at: Specify a timeout for the TEMPORARY_UNAVAILABLE replicas. None for BAD files.
    :param session: The database session in use.

    :returns: True is successful.
    """
    if isinstance(state, string_types):
        rep_state = BadPFNStatus.from_sym(state)
    else:
        rep_state = state

    pfns = clean_surls(pfns)
    for pfn in pfns:
        new_pfn = models.BadPFNs(path=str(pfn), account=account, state=rep_state, reason=reason, expires_at=expires_at)
        new_pfn = session.merge(new_pfn)
        new_pfn.save(session=session, flush=False)

    try:
        session.flush()
    except IntegrityError as error:
        raise exception.RucioException(error.args)
    except DatabaseError as error:
        raise exception.RucioException(error.args)
    except FlushError as error:
        if match('New instance .* with identity key .* conflicts with persistent instance', error.args[0]):
            raise exception.Duplicate('One PFN already exists!')
        raise exception.RucioException(error.args)
    return True 
Example 29
Project: rucio   Author: rucio   File: quarantined_replica.py    License: Apache License 2.0 5 votes vote down vote up
def list_quarantined_replicas(rse_id, limit, worker_number=None, total_workers=None, session=None):
    """
    List RSE Quarantined File replicas.

    :param rse_id: the rse id.
    :param limit: The maximum number of replicas returned.
    :param worker_number:      id of the executing worker.
    :param total_workers:      Number of total workers.
    :param session: The database session in use.

    :returns: a list of dictionary replica.
    """

    query = session.query(models.QuarantinedReplica.path,
                          models.QuarantinedReplica.bytes,
                          models.QuarantinedReplica.scope,
                          models.QuarantinedReplica.name,
                          models.QuarantinedReplica.created_at).\
        filter(models.QuarantinedReplica.rse_id == rse_id)

    # do no delete valid replicas
    stmt = exists(select([1]).prefix_with("/*+ index(REPLICAS REPLICAS_PK) */", dialect='oracle')).\
        where(and_(models.RSEFileAssociation.scope == models.QuarantinedReplica.scope,
                   models.RSEFileAssociation.name == models.QuarantinedReplica.name,
                   models.RSEFileAssociation.rse_id == models.QuarantinedReplica.rse_id))
    query = query.filter(not_(stmt))
    query = filter_thread_work(session=session, query=query, total_threads=total_workers, thread_id=worker_number, hash_variable='path')
    return [{'path': path,
             'rse_id': rse_id,
             'created_at': created_at,
             'scope': scope,
             'name': name,
             'bytes': bytes}
            for path, bytes, scope, name, created_at in query.limit(limit)] 
Example 30
def one_to_many_subquery(self, obj):
        tx_column = option(obj, 'transaction_column_name')

        remote_alias = sa.orm.aliased(self.remote_cls)
        primary_keys = [
            getattr(remote_alias, column.name) for column
            in sa.inspect(remote_alias).mapper.columns
            if column.primary_key and column.name != tx_column
        ]

        return sa.exists(
            sa.select(
                [1]
            ).where(
                sa.and_(
                    getattr(remote_alias, tx_column) <=
                    getattr(obj, tx_column),
                    *[
                        getattr(remote_alias, pk.name) ==
                        getattr(self.remote_cls, pk.name)
                        for pk in primary_keys
                    ]
                )
            ).group_by(
                *primary_keys
            ).having(
                sa.func.max(getattr(remote_alias, tx_column)) ==
                getattr(self.remote_cls, tx_column)
            ).correlate(self.local_cls, self.remote_cls)
        )