Python sqlalchemy.sql.expression.case() Examples

def get_count_of_expired_temporary_dids(rse_id, session=None):
    """
    List expired temporary DIDs.

    :param rse_id: the rse id.
    :param session: The database session in use.

    :returns: a count number.
    """
    is_none = None
    count = session.query(func.count(models.TemporaryDataIdentifier.scope)).\
        with_hint(models.TemporaryDataIdentifier, "INDEX(tmp_dids TMP_DIDS_EXPIRED_AT_IDX)", 'oracle').\
        filter(case([(models.TemporaryDataIdentifier.expired_at != is_none, models.TemporaryDataIdentifier.rse_id), ]) == rse_id).\
        one()

    return count[0] or 0 

def touch_dids(dids, session=None):
    """
    Update the accessed_at timestamp and the access_cnt of the given dids.

    :param replicas: the list of dids.
    :param session: The database session in use.

    :returns: True, if successful, False otherwise.
    """

    now = datetime.utcnow()
    none_value = None
    try:
        for did in dids:
            session.query(models.DataIdentifier).\
                filter_by(scope=did['scope'], name=did['name'], did_type=did['type']).\
                update({'accessed_at': did.get('accessed_at') or now,
                        'access_cnt': case([(models.DataIdentifier.access_cnt == none_value, 1)],
                                           else_=(models.DataIdentifier.access_cnt + 1))},
                       synchronize_session=False)
    except DatabaseError:
        return False

    return True 

def upgrade():
    # re-size existing data if necessary
    identifier_map = table('cisco_csr_identifier_map',
                           column('ipsec_site_conn_id', sa.String(36)))
    ipsec_site_conn_id = identifier_map.columns['ipsec_site_conn_id']

    op.execute(identifier_map.update(values={
        ipsec_site_conn_id: expr.case([(func.length(ipsec_site_conn_id) > 36,
                                      func.substr(ipsec_site_conn_id, 1, 36))],
                                      else_=ipsec_site_conn_id)}))

    # Need to drop foreign key constraint before mysql will allow changes

    with migration.remove_fks_from_table('cisco_csr_identifier_map'):
        op.alter_column(table_name='cisco_csr_identifier_map',
                        column_name='ipsec_site_conn_id',
                        type_=sa.String(36),
                        existing_nullable=False) 

def full_name(cls):
        """
        The person's commonly known full name, following naming order conventions.

        If a person has a nickname, that name becomes the person's full name.

        :return: Person's full name.
        """
        return case(
                [(cls.nick_name != None, cls.nick_name)],
                else_=case(
                    [(cls.order == enums.NameOrderType.middle,
                      cls.first_name + ' ' + cls.middle_name + ' ' + cls.last_name),
                     (cls.order == enums.NameOrderType.eastern,
                      cls.last_name + ' ' + cls.first_name)],
                    else_=cls.first_name + ' ' + cls.last_name
                )) 

def log_param(self, run_id, param):
        with self.ManagedSessionMaker() as session:
            run = self._get_run(run_uuid=run_id, session=session)
            self._check_run_is_active(run)
            # if we try to update the value of an existing param this will fail
            # because it will try to create it with same run_uuid, param key
            try:
                # This will check for various integrity checks for params table.
                # ToDo: Consider prior checks for null, type, param name validations, ... etc.
                self._get_or_create(model=SqlParam, session=session, run_uuid=run_id,
                                    key=param.key, value=param.value)
                # Explicitly commit the session in order to catch potential integrity errors
                # while maintaining the current managed session scope ("commit" checks that
                # a transaction satisfies uniqueness constraints and throws integrity errors
                # when they are violated; "get_or_create()" does not perform these checks). It is
                # important that we maintain the same session scope because, in the case of
                # an integrity error, we want to examine the uniqueness of parameter values using
                # the same database state that the session uses during "commit". Creating a new
                # session synchronizes the state with the database. As a result, if the conflicting
                # parameter value were to be removed prior to the creation of a new session,
                # we would be unable to determine the cause of failure for the first session's
                # "commit" operation.
                session.commit()
            except sqlalchemy.exc.IntegrityError:
                # Roll back the current session to make it usable for further transactions. In the
                # event of an error during "commit", a rollback is required in order to continue
                # using the session. In this case, we re-use the session because the SqlRun, `run`,
                # is lazily evaluated during the invocation of `run.params`.
                session.rollback()
                existing_params = [p.value for p in run.params if p.key == param.key]
                if len(existing_params) > 0:
                    old_value = existing_params[0]
                    raise MlflowException(
                        "Changing param values is not allowed. Param with key='{}' was already"
                        " logged with value='{}' for run ID='{}'. Attempted logging new value"
                        " '{}'.".format(
                            param.key, old_value, run_id, param.value), INVALID_PARAMETER_VALUE)
                else:
                    raise 

def update_replica_lock_counter(rse_id, scope, name, value, session=None):
    """
    Update File replica lock counters.

    :param rse_id: The id of the RSE.
    :param scope: the tag name.
    :param name: The data identifier name.
    :param value: The number of created/deleted locks.
    :param session: The database session in use.

    :returns: True or False.
    """

    # WTF BUG in the mysql-driver: lock_cnt uses the already updated value! ACID? Never heard of it!

    if session.bind.dialect.name == 'mysql':
        rowcount = session.query(models.RSEFileAssociation).\
            filter_by(rse_id=rse_id, scope=scope, name=name).\
            update({'lock_cnt': models.RSEFileAssociation.lock_cnt + value,
                    'tombstone': case([(models.RSEFileAssociation.lock_cnt + value < 0,
                                        datetime.utcnow()), ],
                                      else_=None)},
                   synchronize_session=False)
    else:
        rowcount = session.query(models.RSEFileAssociation).\
            filter_by(rse_id=rse_id, scope=scope, name=name).\
            update({'lock_cnt': models.RSEFileAssociation.lock_cnt + value,
                    'tombstone': case([(models.RSEFileAssociation.lock_cnt + value == 0,
                                        datetime.utcnow()), ],
                                      else_=None)},
                   synchronize_session=False)

    return bool(rowcount) 

def list_expired_temporary_dids(rse_id, limit, worker_number=None, total_workers=None,
                                session=None):
    """
    List expired temporary DIDs.

    :param rse_id: the rse id.
    :param limit: The maximum number of replicas returned.
    :param worker_number:      id of the executing worker.
    :param total_workers:      Number of total workers.
    :param session: The database session in use.

    :returns: a list of dictionary replica.
    """
    is_none = None
    query = session.query(models.TemporaryDataIdentifier.scope,
                          models.TemporaryDataIdentifier.name,
                          models.TemporaryDataIdentifier.path,
                          models.TemporaryDataIdentifier.bytes).\
        with_hint(models.TemporaryDataIdentifier, "INDEX(tmp_dids TMP_DIDS_EXPIRED_AT_IDX)", 'oracle').\
        filter(case([(models.TemporaryDataIdentifier.expired_at != is_none, models.TemporaryDataIdentifier.rse_id), ]) == rse_id)

    query = filter_thread_work(session=session, query=query, total_threads=total_workers, thread_id=worker_number, hash_variable='name')

    return [{'path': path,
             'rse_id': rse_id,
             'scope': scope,
             'name': name,
             'bytes': bytes}
            for scope, name, path, bytes in query.limit(limit)] 

def deprecated(cls):
        return case([(cls.name in BAD_CIPHERS, True)], else_=False) 

def expired(cls):
        return case([(cls.not_after <= arrow.utcnow(), True)], else_=False) 

def revoked(cls):
        return case([(cls.status == "revoked", True)], else_=False) 

def has_private_key(cls):
        return case([(cls.private_key.is_(None), True)], else_=False) 

def in_rotation_window(cls):
        """
        Determines if a certificate is available for rotation based
        on the rotation policy associated.
        :return:
        """
        return case(
            [(extract("day", cls.not_after - func.now()) <= RotationPolicy.days, True)],
            else_=False,
        ) 

def name(cls):
        return case(
            [(
                and_(cls.last_name != None, cls.last_name != ""), 
                cls.last_name + " " + cls.first_name
            )],
            else_=cls.first_name
        ) 

def queue_get_next(
        self, manager, available_programs, available_procedures, limit=100, tag=None, as_json=True
    ) -> List[TaskRecord]:
        """Done in a transaction"""

        # Figure out query, tagless has no requirements

        proc_filt = TaskQueueORM.procedure.in_([p.lower() for p in available_procedures])
        none_filt = TaskQueueORM.procedure == None  # lgtm [py/test-equals-none]

        order_by = []
        if tag is not None:
            if isinstance(tag, str):
                tag = [tag]
            # task_order = expression_case([(TaskQueueORM.tag == t, num) for num, t in enumerate(tag)])
            # order_by.append(task_order)

        order_by.extend([TaskQueueORM.priority.desc(), TaskQueueORM.created_on])
        queries = []
        if tag is not None:
            for t in tag:
                query = format_query(TaskQueueORM, status=TaskStatusEnum.waiting, program=available_programs, tag=t)
                query.append(or_(proc_filt, none_filt))
                queries.append(query)
        else:
            query = format_query(TaskQueueORM, status=TaskStatusEnum.waiting, program=available_programs)
            query.append((or_(proc_filt, none_filt)))
            queries.append(query)
        new_limit = limit
        ids = []
        found = []
        with self.session_scope() as session:
            for q in queries:
                if new_limit == 0:
                    break
                query = session.query(TaskQueueORM).filter(*q).order_by(*order_by).limit(new_limit)
                # from sqlalchemy.dialects import postgresql
                # print(query.statement.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True}))
                new_items = query.all()
                found.extend(new_items)
                new_limit = limit - len(new_items)
                ids.extend([x.id for x in new_items])
            update_fields = {"status": TaskStatusEnum.running, "modified_on": dt.utcnow(), "manager": manager}
            # # Bulk update operation in SQL
            update_count = (
                session.query(TaskQueueORM)
                .filter(TaskQueueORM.id.in_(ids))
                .update(update_fields, synchronize_session=False)
            )

            if as_json:
                # avoid another trip to the DB to get the updated values, set them here
                found = [TaskRecord(**task.to_dict(exclude=update_fields.keys()), **update_fields) for task in found]
            session.commit()

        if update_count != len(found):
            self.logger.warning("QUEUE: Number of found projects does not match the number of updated projects.")

        return found 

def list_rebalance_rule_candidates(rse, mode=None, session=None):
    """
    List the rebalance rule candidates based on the agreed on specification

    :param rse:          RSE of the source.
    :param mode:         Rebalancing mode.
    :param session:      DB Session.
    """

    rse_id = get_rse_id(rse)

    # dumps can be applied only for decommission since the dumps doesn't contain info from dids
    if mode == 'decommission':
        return _list_rebalance_rule_candidates_dump(rse, mode)

    # the rest is done with sql query
    from_date = datetime.utcnow() + timedelta(days=60)
    to_date = datetime.now() - timedelta(days=60)
    allowed_accounts = [InternalAccount(a) for a in ('panda', 'root', 'ddmadmin')]
    allowed_grouping = [RuleGrouping.DATASET, RuleGrouping.ALL]
    external_dsl = aliased(models.DatasetLock)
    count_locks = select([func.count()]).where(and_(external_dsl.scope == models.DatasetLock.scope,
                                                    external_dsl.name == models.DatasetLock.name,
                                                    external_dsl.rse_id == models.DatasetLock.rse_id)).as_scalar()
    query = session.query(models.DatasetLock.scope,
                          models.DatasetLock.name,
                          models.ReplicationRule.id,
                          models.ReplicationRule.rse_expression,
                          models.ReplicationRule.subscription_id,
                          models.DataIdentifier.bytes,
                          models.DataIdentifier.length,
                          case([(or_(models.DatasetLock.length < 1, models.DatasetLock.length.is_(None)), 0)],
                               else_=cast(models.DatasetLock.bytes / models.DatasetLock.length, Integer))).\
        join(models.ReplicationRule, models.ReplicationRule.id == models.DatasetLock.rule_id).\
        join(models.DataIdentifier, and_(models.DatasetLock.scope == models.DataIdentifier.scope, models.DatasetLock.name == models.DataIdentifier.name)).\
        filter(models.DatasetLock.rse_id == rse_id).\
        filter(or_(models.ReplicationRule.expires_at > from_date, models.ReplicationRule.expires_at.is_(None))).\
        filter(models.ReplicationRule.created_at < to_date).\
        filter(models.ReplicationRule.account.in_(allowed_accounts)).\
        filter(models.ReplicationRule.state == RuleState.OK).\
        filter(models.ReplicationRule.did_type == DIDType.DATASET).\
        filter(models.ReplicationRule.copies == 1).\
        filter(models.ReplicationRule.child_rule_id.is_(None)).\
        filter(models.ReplicationRule.grouping.in_(allowed_grouping)).\
        filter(models.DataIdentifier.bytes.isnot(None)).\
        filter(models.DataIdentifier.is_open == 0).\
        filter(models.DataIdentifier.did_type == DIDType.DATASET).\
        filter(case([(or_(models.DatasetLock.length < 1, models.DatasetLock.length.is_(None)), 0)],
                    else_=cast(models.DatasetLock.bytes / models.DatasetLock.length, Integer)) > 1000000000).\
        filter(count_locks == 1)
    summary = query.order_by(case([(or_(models.DatasetLock.length < 1, models.DatasetLock.length.is_(None)), 0)],
                                  else_=cast(models.DatasetLock.bytes / models.DatasetLock.length, Integer)),
                             models.DatasetLock.accessed_at).all()
    return summary 

def _get_orderby_clauses(order_by_list, session):
    """Sorts a set of runs based on their natural ordering and an overriding set of order_bys.
    Runs are naturally ordered first by start time descending, then by run id for tie-breaking.
    """

    clauses = []
    ordering_joins = []
    clause_id = 0
    # contrary to filters, it is not easily feasible to separately handle sorting
    # on attributes and on joined tables as we must keep all clauses in the same order
    if order_by_list:
        for order_by_clause in order_by_list:
            clause_id += 1
            (key_type, key, ascending) = SearchUtils.parse_order_by_for_search_runs(order_by_clause)
            if SearchUtils.is_attribute(key_type, '='):
                order_value = getattr(SqlRun, SqlRun.get_attribute_name(key))
            else:
                if SearchUtils.is_metric(key_type, '='):  # any valid comparator
                    entity = SqlLatestMetric
                elif SearchUtils.is_tag(key_type, '='):
                    entity = SqlTag
                elif SearchUtils.is_param(key_type, '='):
                    entity = SqlParam
                else:
                    raise MlflowException("Invalid identifier type '%s'" % key_type,
                                          error_code=INVALID_PARAMETER_VALUE)

                # build a subquery first because we will join it in the main request so that the
                # metric we want to sort on is available when we apply the sorting clause
                subquery = session \
                    .query(entity) \
                    .filter(entity.key == key) \
                    .subquery()

                ordering_joins.append(subquery)
                order_value = subquery.c.value

            # sqlite does not support NULLS LAST expression, so we sort first by
            # presence of the field (and is_nan for metrics), then by actual value
            # As the subqueries are created independently and used later in the
            # same main query, the CASE WHEN columns need to have unique names to
            # avoid ambiguity
            if SearchUtils.is_metric(key_type, '='):
                clauses.append(sql.case([
                    (subquery.c.is_nan.is_(True), 1),
                    (order_value.is_(None), 1)
                ], else_=0).label('clause_%s' % clause_id))
            else:  # other entities do not have an 'is_nan' field
                clauses.append(sql.case([(order_value.is_(None), 1)], else_=0)
                               .label('clause_%s' % clause_id))

            if ascending:
                clauses.append(order_value)
            else:
                clauses.append(order_value.desc())

    clauses.append(SqlRun.start_time.desc())
    clauses.append(SqlRun.run_uuid)
    return clauses, ordering_joins 

def setup(db):
    class Node(db.Model):
        class Meta:
            repr = ('id', 'name', 'path')

        slug = db.Column(db.String, index=True, unique=True)

        parent_id = db.foreign_key('Node', nullable=True)
        parent = db.relationship('Node', back_populates='children',
                                 remote_side='Node.id')
        children = db.relationship('Node', back_populates='parent')

        mv = db.relationship('NodeMV', uselist=False, foreign_keys='NodeMV.id',
                             primaryjoin='Node.id == NodeMV.id', viewonly=True)
        depth = db.association_proxy('mv', 'depth')
        path = db.association_proxy('mv', 'path')

    class NodeMV(db.MaterializedView):
        class Meta:
            mv_for = 'Node'

        @classmethod
        def selectable(cls):
            _cte = (db.select([Node.id.label('id'),
                               literal(0).label('depth'),
                               literal('/').label('path')])
                    .where(Node.parent_id == None)
                    .cte(name='nodes_cte', recursive=True))
            _union = _cte.union_all(
                db.select([
                    Node.id.label('id'),
                    label('depth', _cte.c.depth + 1),
                    label('path', case([
                        # tuple(if condition, then value),
                        (_cte.c.depth == 0, _cte.c.path + Node.slug),
                    ], else_=_cte.c.path + '/' + Node.slug)),
                ]).select_from(db.join(_cte, Node, _cte.c.id == Node.parent_id))
            )
            return db.select([_union])

    unchained.sqlalchemy_bundle.models = UnchainedModelRegistry().finalize_mappings()
    db.create_all()

    class NodeManager(ModelManager):
        class Meta:
            model = Node

    return Node, NodeMV, NodeManager() 

def touch_replica(replica, session=None):
    """
    Update the accessed_at timestamp of the given file replica/did but don't wait if row is locked.

    :param replica: a dictionary with the information of the affected replica.
    :param session: The database session in use.

    :returns: True, if successful, False otherwise.
    """
    try:
        accessed_at, none_value = replica.get('accessed_at') or datetime.utcnow(), None

        session.query(models.RSEFileAssociation).\
            filter_by(rse_id=replica['rse_id'], scope=replica['scope'], name=replica['name']).\
            with_hint(models.RSEFileAssociation, "index(REPLICAS REPLICAS_PK)", 'oracle').\
            with_for_update(nowait=True).one()

        session.query(models.RSEFileAssociation).filter_by(rse_id=replica['rse_id'], scope=replica['scope'], name=replica['name']).\
            with_hint(models.RSEFileAssociation, "index(REPLICAS REPLICAS_PK)", 'oracle').\
            update({'accessed_at': accessed_at,
                    'tombstone': case([(and_(models.RSEFileAssociation.tombstone != none_value,
                                             models.RSEFileAssociation.tombstone != OBSOLETE),
                                        accessed_at)],
                                      else_=models.RSEFileAssociation.tombstone)},
                   synchronize_session=False)

        session.query(models.DataIdentifier).\
            filter_by(scope=replica['scope'], name=replica['name'], did_type=DIDType.FILE).\
            with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').\
            with_for_update(nowait=True).one()

        session.query(models.DataIdentifier).\
            filter_by(scope=replica['scope'], name=replica['name'], did_type=DIDType.FILE).\
            with_hint(models.DataIdentifier, "INDEX(DIDS DIDS_PK)", 'oracle').\
            update({'accessed_at': accessed_at}, synchronize_session=False)

    except DatabaseError:
        return False
    except NoResultFound:
        return True

    return True 

def update_replicas_states(replicas, nowait=False, add_tombstone=False, session=None):
    """
    Update File replica information and state.

    :param replicas:        The list of replicas.
    :param nowait:          Nowait parameter for the for_update queries.
    :param add_tombstone:   To set a tombstone in case there is no lock on the replica.
    :param session:         The database session in use.
    """

    for replica in replicas:
        query = session.query(models.RSEFileAssociation).filter_by(rse_id=replica['rse_id'], scope=replica['scope'], name=replica['name'])
        lock_cnt = 0
        try:
            if nowait:
                rep = query.with_for_update(nowait=True).one()
            else:
                rep = query.one()
            lock_cnt = rep.lock_cnt
        except NoResultFound:
            # remember scope, name and rse
            raise exception.ReplicaNotFound("No row found for scope: %s name: %s rse: %s" % (replica['scope'], replica['name'], get_rse_name(replica['rse_id'], session=session)))

        if isinstance(replica['state'], string_types):
            replica['state'] = ReplicaState.from_string(replica['state'])

        values = {'state': replica['state']}
        if replica['state'] == ReplicaState.BEING_DELETED:
            query = query.filter_by(lock_cnt=0)
            # Exclude replicas use as sources
            stmt = exists([1]).where(and_(models.RSEFileAssociation.scope == models.Source.scope,
                                          models.RSEFileAssociation.name == models.Source.name,
                                          models.RSEFileAssociation.rse_id == models.Source.rse_id))
            query = query.filter(not_(stmt))
            values['tombstone'] = OBSOLETE
        elif replica['state'] == ReplicaState.AVAILABLE:
            rucio.core.lock.successful_transfer(scope=replica['scope'], name=replica['name'], rse_id=replica['rse_id'], nowait=nowait, session=session)
            # If No locks we set a tombstone in the future
            if add_tombstone and lock_cnt == 0:
                set_tombstone(rse_id=replica['rse_id'], scope=replica['scope'], name=replica['name'], tombstone=datetime.utcnow() + timedelta(hours=2), session=session)

        elif replica['state'] == ReplicaState.UNAVAILABLE:
            rucio.core.lock.failed_transfer(scope=replica['scope'], name=replica['name'], rse_id=replica['rse_id'],
                                            error_message=replica.get('error_message', None),
                                            broken_rule_id=replica.get('broken_rule_id', None),
                                            broken_message=replica.get('broken_message', None),
                                            nowait=nowait, session=session)
            # If No locks we set a tombstone in the future
            if add_tombstone and lock_cnt == 0:
                set_tombstone(rse_id=replica['rse_id'], scope=replica['scope'], name=replica['name'], tombstone=datetime.utcnow() + timedelta(hours=2), session=session)
        elif replica['state'] == ReplicaState.TEMPORARY_UNAVAILABLE:
            query = query.filter(or_(models.RSEFileAssociation.state == ReplicaState.AVAILABLE, models.RSEFileAssociation.state == ReplicaState.TEMPORARY_UNAVAILABLE))

        if 'path' in replica and replica['path']:
            values['path'] = replica['path']

        if not query.update(values, synchronize_session=False):
            if 'rse' not in replica:
                replica['rse'] = get_rse_name(rse_id=replica['rse_id'], session=session)
            raise exception.UnsupportedOperation('State %(state)s for replica %(scope)s:%(name)s on %(rse)s cannot be updated' % replica)
    return True 

def list_unlocked_replicas(rse_id, limit, bytes=None, worker_number=None, total_workers=None, delay_seconds=0, session=None):
    """
    List RSE File replicas with no locks.

    :param rse_id: the rse id.
    :param bytes: the amount of needed bytes.
    :param session: The database session in use.

    :returns: a list of dictionary replica.
    """

    # filter(models.RSEFileAssociation.state != ReplicaState.BEING_DELETED).\
    none_value = None  # Hack to get pep8 happy...
    query = session.query(models.RSEFileAssociation.scope, models.RSEFileAssociation.name, models.RSEFileAssociation.path, models.RSEFileAssociation.bytes, models.RSEFileAssociation.tombstone, models.RSEFileAssociation.state).\
        with_hint(models.RSEFileAssociation, "INDEX_RS_ASC(replicas REPLICAS_TOMBSTONE_IDX)  NO_INDEX_FFS(replicas REPLICAS_TOMBSTONE_IDX)", 'oracle').\
        filter(models.RSEFileAssociation.tombstone < datetime.utcnow()).\
        filter(models.RSEFileAssociation.lock_cnt == 0).\
        filter(case([(models.RSEFileAssociation.tombstone != none_value, models.RSEFileAssociation.rse_id), ]) == rse_id).\
        filter(or_(models.RSEFileAssociation.state.in_((ReplicaState.AVAILABLE, ReplicaState.UNAVAILABLE, ReplicaState.BAD)),
                   and_(models.RSEFileAssociation.state == ReplicaState.BEING_DELETED, models.RSEFileAssociation.updated_at < datetime.utcnow() - timedelta(seconds=delay_seconds)))).\
        order_by(models.RSEFileAssociation.tombstone)

    # do no delete files used as sources
    stmt = exists(select([1]).prefix_with("/*+ INDEX(requests REQUESTS_SCOPE_NAME_RSE_IDX) */", dialect='oracle')).\
        where(and_(models.RSEFileAssociation.scope == models.Request.scope,
                   models.RSEFileAssociation.name == models.Request.name))
    query = query.filter(not_(stmt))
    query = filter_thread_work(session=session, query=query, total_threads=total_workers, thread_id=worker_number, hash_variable='name')
    needed_space = bytes
    total_bytes, total_files = 0, 0
    rows = []
    for (scope, name, path, bytes, tombstone, state) in query.yield_per(1000):
        if state != ReplicaState.UNAVAILABLE:

            if tombstone != OBSOLETE and needed_space is not None and total_bytes > needed_space:
                break
            total_bytes += bytes

            total_files += 1
            if total_files > limit:
                break

        rows.append({'scope': scope, 'name': name, 'path': path,
                     'bytes': bytes, 'tombstone': tombstone,
                     'state': state})
    return rows