Python boto3.dynamodb.conditions.Attr() Examples

The following are 28 code examples of boto3.dynamodb.conditions.Attr(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module boto3.dynamodb.conditions , or try the search function .
Example #1
Source File: dynamo_connection.py    From aws-syndicate with Apache License 2.0 7 votes vote down vote up
def get_items_with_attribute_value(self, table_name, attr_name,
                                       attr_value):
        """ Get all items from table, which have specified attribute with
        specified value.

        :type table_name: str
        :type attr_name: str
        :param attr_value: attr value in table
        :return: list(if items exist)
        """
        table = self.get_table_by_name(table_name)
        result = []
        response = self.scan(table=table,
                             filter_expr=Attr(attr_name).eq(attr_value))
        do_continue = response.get('LastEvaluatedKey')
        if 'Items' in response:
            result.extend(response['Items'])
        while do_continue:
            response = self.scan(table=table, token=do_continue,
                                 filter_expr=Attr(attr_name).eq(attr_value))
            do_continue = response.get('LastEvaluatedKey')
            if 'Items' in response:
                result.extend(response['Items'])
        return result 
Example #2
Source File: task_tracking_table.py    From aws-ops-automator with Apache License 2.0 6 votes vote down vote up
def get_task_items_for_job(self, task_group):

        job_tasks = []

        args = {
            "Select": "ALL_ATTRIBUTES",
            "ProjectionExpression": handlers.TASK_TR_ID,
            "ConsistentRead": True,
            "FilterExpression": Attr(handlers.TASK_TR_GROUP).eq(task_group)
        }

        while True:
            resp = self._action_table.scan_with_retries(**args)

            for item in resp.get("Items", []):
                job_tasks.append(item)
            if "LastEvaluatedKey" in resp:
                args["ExclusiveStartKey"] = resp["LastEvaluatedKey"]
            else:
                break

        return job_tasks 
Example #3
Source File: ramps.py    From motorway with Apache License 2.0 6 votes vote down vote up
def claim_shard(self, shard_id):
        """
        Atomically update the shard in DynamoDB

        :param shard_id:
        :return: bool
        """
        shard_election_logger.info("Claiming shard %s" % shard_id)
        try:
            control_record = self.control_table.get_item(Key={'shard_id': shard_id})['Item']
        except KeyError:
            raise NoItemsReturned()
        control_record['worker_id'] = self.worker_id
        control_record['heartbeat'] = 0
        try:
            self.control_table.put_item(Item=control_record,
                                        ConditionExpression=Attr('shard_id').eq(shard_id) & Attr('checkpoint').eq(control_record['checkpoint'])
                                        # ensure that the record was not changed between the get and put.
                                        )
        except self.dynamodb_client.exceptions.ConditionalCheckFailedException:  # Someone else edited the record
            shard_election_logger.debug("Failed to claim %s to %s" % (shard_id, self.worker_id))
            return False
        return True 
Example #4
Source File: db.py    From chalice-workshop with Apache License 2.0 6 votes vote down vote up
def list_media_files(self, startswith=None, media_type=None, label=None):
        scan_params = {}
        filter_expression = None
        if startswith is not None:
            filter_expression = self._add_to_filter_expression(
                filter_expression, Attr('name').begins_with(startswith)
            )
        if media_type is not None:
            filter_expression = self._add_to_filter_expression(
                filter_expression, Attr('type').eq(media_type)
            )
        if label is not None:
            filter_expression = self._add_to_filter_expression(
                filter_expression, Attr('labels').contains(label)
            )
        if filter_expression:
            scan_params['FilterExpression'] = filter_expression
        response = self._table.scan(**scan_params)
        return response['Items'] 
Example #5
Source File: dyndbmutex.py    From dyndb-mutex with Apache License 2.0 6 votes vote down vote up
def clear_lock_item(self, lockname, caller):
        try:
            self.get_table().put_item(
                Item={
                    'lockname': lockname,
                    'expire_ts': 0,
                    'holder': NO_HOLDER
                },
                ConditionExpression=Attr("holder").eq(caller) | Attr('lockname').not_exists()
            )
        except botocore.exceptions.ClientError as e:
            if e.response['Error']['Code'] == 'ConditionalCheckFailedException':
                logger.warning("clear_lock_item: lockname=" + lockname + ", caller=" + caller +
                             " release failed")
                return False
        logger.debug("clear_lock_item: lockname=" + lockname + ", caller=" + caller + " release succeeded")
        return True 
Example #6
Source File: dyndbmutex.py    From dyndb-mutex with Apache License 2.0 6 votes vote down vote up
def prune_expired(self, lockname, caller):
        now = timestamp_millis()
        logger.debug("Prune: lockname=" + lockname + ", caller=" + caller +
                     ", Time now is %s" + str(now))
        try:
            self.get_table().put_item(
                Item={
                    'lockname': lockname,
                    'expire_ts': 0,
                    'holder': NO_HOLDER
                },
                ConditionExpression=Attr("expire_ts").lt(now) | Attr('lockname').not_exists()
            )
        except botocore.exceptions.ClientError as e:
            if e.response['Error']['Code'] == 'ConditionalCheckFailedException':
                logger.info("Prune: lockname=" + lockname + ", caller=" + caller +
                             " Prune failed")
                return False
        logger.debug("Prune: lockname=" + lockname + ", caller=" + caller + " Prune succeeded")
        return True 
Example #7
Source File: db.py    From chalice-workshop with Apache License 2.0 6 votes vote down vote up
def list_media_files(self, startswith=None, media_type=None, label=None):
        scan_params = {}
        filter_expression = None
        if startswith is not None:
            filter_expression = self._add_to_filter_expression(
                filter_expression, Attr('name').begins_with(startswith)
            )
        if media_type is not None:
            filter_expression = self._add_to_filter_expression(
                filter_expression, Attr('type').eq(media_type)
            )
        if label is not None:
            filter_expression = self._add_to_filter_expression(
                filter_expression, Attr('labels').contains(label)
            )
        if filter_expression:
            scan_params['FilterExpression'] = filter_expression
        response = self._table.scan(**scan_params)
        return response['Items'] 
Example #8
Source File: support.py    From awslimits with Apache License 2.0 6 votes vote down vote up
def update_limit_value(limit_type):
    service, limit_name = limit_type.split(NAME_SEPARATOR)
    checker = get_aws_limit_checker()
    limits = checker.get_limits(use_ta=settings.PREMIUM_ACCOUNT)
    default_limit = limits[service][limit_name].default_limit

    dynamodb = get_boto_resource('dynamodb')
    tickets_table = get_tickets_table()

    tickets = tickets_table.scan(
        FilterExpression=Attr('limit_type').eq(limit_type)
    )['Items']
    if tickets:
        max_value = max(ticket['limit_value'] for ticket in tickets)
    else:
        max_value = 0

    max_value = max([max_value, default_limit])
    update_dynamodb_limit_value(limit_type, max_value) 
Example #9
Source File: support.py    From awslimits with Apache License 2.0 6 votes vote down vote up
def get_recently_sent_alerts(limits):
    table = create_or_get_table(
        table_name=SENT_ALERTS_TABLE_NAME,
        attribute_definitions=[
            {
                'AttributeName': 'limit_name',
                'AttributeType': 'S',
            },
        ],
        key_schema=[
            {
                'AttributeName': 'limit_name',
                'KeyType': 'HASH'
            },
        ],
    )

    three_days_ago_ts = Decimal((datetime.utcnow() - timedelta(days=3)).strftime('%s'))
    alerts = table.scan(
        FilterExpression=Attr('alert_sent').gt(three_days_ago_ts)
    )['Items']
    return [alert['limit_name'] for alert in alerts] 
Example #10
Source File: db.py    From chalice-workshop with Apache License 2.0 6 votes vote down vote up
def list_media_files(self, startswith=None, media_type=None, label=None):
        scan_params = {}
        filter_expression = None
        if startswith is not None:
            filter_expression = self._add_to_filter_expression(
                filter_expression, Attr('name').begins_with(startswith)
            )
        if media_type is not None:
            filter_expression = self._add_to_filter_expression(
                filter_expression, Attr('type').eq(media_type)
            )
        if label is not None:
            filter_expression = self._add_to_filter_expression(
                filter_expression, Attr('labels').contains(label)
            )
        if filter_expression:
            scan_params['FilterExpression'] = filter_expression
        response = self._table.scan(**scan_params)
        return response['Items'] 
Example #11
Source File: db.py    From chalice-workshop with Apache License 2.0 6 votes vote down vote up
def list_media_files(self, startswith=None, media_type=None, label=None):
        scan_params = {}
        filter_expression = None
        if startswith is not None:
            filter_expression = self._add_to_filter_expression(
                filter_expression, Attr('name').begins_with(startswith)
            )
        if media_type is not None:
            filter_expression = self._add_to_filter_expression(
                filter_expression, Attr('type').eq(media_type)
            )
        if label is not None:
            filter_expression = self._add_to_filter_expression(
                filter_expression, Attr('labels').contains(label)
            )
        if filter_expression:
            scan_params['FilterExpression'] = filter_expression
        response = self._table.scan(**scan_params)
        return response['Items'] 
Example #12
Source File: dynamo_connection.py    From aws-syndicate with Apache License 2.0 6 votes vote down vote up
def get_all_items(self, table_name, filter_expr=None):
        """ Get all items from table.

        :type table_name: str
        :type filter_expr: Attr
        :return list(if items exist)
        """
        table = self.get_table_by_name(table_name)
        response = self.scan(table=table, filter_expr=filter_expr)
        do_continue = response.get('LastEvaluatedKey')
        items = []
        if 'Items' in response:
            items.extend(response['Items'])
        while do_continue:
            response = self.scan(table=table, token=do_continue,
                                 filter_expr=filter_expr)
            do_continue = response.get('LastEvaluatedKey')
            if 'Items' in response:
                items.extend(response['Items'])
        return items 
Example #13
Source File: app.py    From aws-media-insights-engine with Apache License 2.0 6 votes vote down vote up
def list_workflows_by_stage(StageName):
    """ List all workflow defintions that contain a stage

    Returns:
        A list of workflow definitions.

    Raises:
        200: All workflows returned sucessfully.
        500: Internal server error
    """

    table = DYNAMO_RESOURCE.Table(WORKFLOW_TABLE_NAME)

    response = table.scan(
        FilterExpression=Attr("Stages."+StageName).exists(),
        ConsistentRead=True
    )
    workflows = response['Items']
    while 'LastEvaluatedKey' in response:
        response = table.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
        workflows.extend(response['Items'])

    return workflows 
Example #14
Source File: tile.py    From sso-dashboard with Mozilla Public License 2.0 5 votes vote down vote up
def sync_config(self):
        self.connect_dynamodb()
        results = self.dynamodb.scan(FilterExpression=Attr("name").exists())
        return results.get("Items", None) 
Example #15
Source File: meta.py    From aws-dynamodb-encryption-python with Apache License 2.0 5 votes vote down vote up
def _save_materials(self, material_name, version, encryption_key, signing_key):
        # type: (Text, int, JceNameLocalDelegatedKey, JceNameLocalDelegatedKey) -> None
        """Save materials to the table, raising an error if the version already exists.

        :param str material_name: Material to locate
        :param int version: Version of material to locate
        :raises VersionAlreadyExistsError: if the specified version already exists
        """
        _LOGGER.debug('Saving material "%s" version %d to MetaStore table', material_name, version)
        item = {
            MetaStoreAttributeNames.PARTITION.value: material_name,
            MetaStoreAttributeNames.SORT.value: version,
            MetaStoreAttributeNames.MATERIAL_TYPE_VERSION.value: MetaStoreValues.MATERIAL_TYPE_VERSION.value,
            MetaStoreAttributeNames.ENCRYPTION_ALGORITHM.value: encryption_key.algorithm,
            MetaStoreAttributeNames.ENCRYPTION_KEY.value: Binary(encryption_key.key),
            MetaStoreAttributeNames.INTEGRITY_ALGORITHM.value: signing_key.algorithm,
            MetaStoreAttributeNames.INTEGRITY_KEY.value: Binary(signing_key.key),
        }
        try:
            self._encrypted_table.put_item(
                Item=item,
                ConditionExpression=(
                    Attr(MetaStoreAttributeNames.PARTITION.value).not_exists()
                    & Attr(MetaStoreAttributeNames.SORT.value).not_exists()
                ),
            )
        except botocore.exceptions.ClientError as error:
            if error.response["Error"]["Code"] == "ConditionalCheckFailedException":
                raise VersionAlreadyExistsError('Version already exists: "{}#{}"'.format(material_name, version)) 
Example #16
Source File: db.py    From chalice-workshop with Apache License 2.0 5 votes vote down vote up
def list_media_files(self, startswith=None, media_type=None, label=None):
        scan_params = {}
        filter_expression = None
        if startswith is not None:
            filter_expression = self._add_to_filter_expression(
                filter_expression, Attr('name').begins_with(startswith)
            )
        if media_type is not None:
            filter_expression = self._add_to_filter_expression(
                filter_expression, Attr('type').eq(media_type)
            )
        if label is not None:
            filter_expression = self._add_to_filter_expression(
                filter_expression, Attr('labels').contains(label)
            )
        if filter_expression:
            scan_params['FilterExpression'] = filter_expression
        response = self._table.scan(**scan_params)
        return response['Items'] 
Example #17
Source File: app.py    From aws-media-insights-engine with Apache License 2.0 5 votes vote down vote up
def list_workflows_by_operator(OperatorName):
    """ List all workflow defintions that contain an operator

    Returns:
        A list of workflow definitions.

    Raises:
        200: All workflows returned sucessfully.
        500: Internal server error
    """

    table = DYNAMO_RESOURCE.Table(WORKFLOW_TABLE_NAME)

    response = table.scan(
        FilterExpression=Attr("Operations").contains(OperatorName),
        ConsistentRead=True
    )
    workflows = response['Items']
    while 'LastEvaluatedKey' in response:
        response = table.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
        workflows.extend(response['Items'])

    return workflows 
Example #18
Source File: dynamo_connection.py    From aws-syndicate with Apache License 2.0 5 votes vote down vote up
def get_items_with_attribute_contains(self, table_name, attr_name, val):
        """ Get all items from table, which have specified attribute.

        :type table_name: str
        :type attr_name: str
        :type val: any
        :return list(if items exist)
        """
        return self.get_all_items(table_name, Attr(attr_name).contains(val)) 
Example #19
Source File: dynamo_connection.py    From aws-syndicate with Apache License 2.0 5 votes vote down vote up
def for_each_item_in_interval(self, start_interval, end_interval,
                                  interval_time, table_name, func,
                                  *args, **kwargs):
        """ Calls a function for each item in the table.

        :type start_interval: int (timestamp)
        :type end_interval: int (timestamp)
        :type interval_time: int (timestamp)
        :type table_name: str
        :param func: Function to call for each item.
                     It must accept the item as the first argument.
        :type func: function
        :param args: any args the function takes after the item
        :param kwargs: any kwargs the func takes after the item and args
        """
        table = self.get_table_by_name(table_name)
        start = start_interval
        count = 0
        while start < end_interval:
            response = self.scan(table=table,
                                 filter_expr=Attr('d').eq(start))
            do_continue = response.get('LastEvaluatedKey')
            if 'Items' in response:
                for each in response['Items']:
                    count += 1
                    func(each, *args, **kwargs)
            while do_continue:
                response = self.scan(table=table, token=do_continue,
                                     filter_expr=Attr('d').eq(start))
                do_continue = response.get('LastEvaluatedKey')
                if 'Items' in response:
                    for each in response['Items']:
                        count += 1
                        func(each, *args, **kwargs)
            start += interval_time 
Example #20
Source File: dynamo_connection.py    From aws-syndicate with Apache License 2.0 5 votes vote down vote up
def scan(self, table_name=None, table=None, token=None,
             filter_expr=None, limit=None, attr_to_select='ALL_ATTRIBUTES'):
        """ DynamoDB table scan with consistent read and custom retry.

        :type table_name: str
        :type table: Table
        :type token: dict
        :type filter_expr: Attr
        :type limit: int
        :type attr_to_select: str
        """
        if table and table_name:
            raise ValueError("'table' OR 'table_name' must be set.")
        elif table:
            pass  # just use Table object
        elif table_name:
            table = self.get_table_by_name(table_name)
        else:
            raise ValueError("'table' or 'table_name' must be set.")

        params = {'ConsistentRead': True, 'Select': attr_to_select}
        if token:
            params['ExclusiveStartKey'] = token
        if filter_expr:
            params['FilterExpression'] = filter_expr
        if limit:
            params['Limit'] = limit

        return table.scan(**params) 
Example #21
Source File: support.py    From awslimits with Apache License 2.0 5 votes vote down vote up
def get_pending_tickets():
    table = get_tickets_table()
    cases = table.scan(
        FilterExpression=Attr('limit_type').eq('unknown') & Attr('body').ne('N/A')
    )['Items']
    cases = sorted(cases, key=lambda case: case['display_id'], reverse=True)
    return cases 
Example #22
Source File: dyndbmutex.py    From dyndb-mutex with Apache License 2.0 5 votes vote down vote up
def write_lock_item(self, lockname, caller, waitms):
        expire_ts = timestamp_millis() + waitms
        ttl = expire_ts + self.ttl_minutes*60
        logger.debug("Write_item: lockname=" + lockname + ", caller=" +
                     caller + ", Expire time is " + str(expire_ts))
        try:
            self.get_table().put_item(
                Item={
                    'lockname': lockname,
                    'expire_ts': expire_ts,
                    'holder': caller,
                    'ttl': ttl
                },
                # TODO: adding Attr("holder").eq(caller) should make it re-entrant
                ConditionExpression=Attr("holder").eq(NO_HOLDER) | Attr('lockname').not_exists()
            )
        except botocore.exceptions.ClientError as e:
            if e.response['Error']['Code'] == 'ConditionalCheckFailedException':
                logger.info("Write_item: lockname=" + lockname +
                             ", caller=" + caller + ", lock is being held")
                return False
        logger.debug("Write_item: lockname=" + lockname +
                     ", caller=" + caller + ", lock is acquired")
        return True 
Example #23
Source File: db.py    From kev with GNU General Public License v3.0 5 votes vote down vote up
def parse_filters(self, filters, doc_class):
        index_name = None
        filter_expression_list = []
        query_params = {}
        for idx, filter in enumerate(filters):
            prop_name, prop_value = filter.split(':')[3:5]
            if idx == 0:
                prop = doc_class()._base_properties[prop_name]
                index_name = prop.kwargs.get(self.index_field_name, None) or \
                             self.default_index_name.format(prop_name)
                query_params['KeyConditionExpression'] = Key(prop_name).eq(prop_value)
            else:
                filter_expression_list.append(Attr(prop_name).eq(prop_value))
        if len(filter_expression_list) > 1:
            query_params['FilterExpression'] = And(*filter_expression_list)
        elif len(filter_expression_list) == 1:
            query_params['FilterExpression'] = filter_expression_list[0]
        if index_name != '_id':
            query_params['IndexName'] = index_name
        return query_params 
Example #24
Source File: credstash.py    From credstash with Apache License 2.0 5 votes vote down vote up
def putSecret(name, secret, version="", kms_key="alias/credstash",
              region=None, table="credential-store", context=None,
              digest=DEFAULT_DIGEST, comment="", kms=None, dynamodb=None, 
              kms_region=None, **kwargs):
    '''
    put a secret called `name` into the secret-store,
    protected by the key kms_key
    '''
    if not context:
        context = {}

    if dynamodb is None or kms is None:
        session = get_session(**kwargs)
        if dynamodb is None:
            dynamodb = session.resource('dynamodb', region_name=region)
        if kms is None:
            kms = session.client('kms', region_name=kms_region or region)

    key_service = KeyService(kms, kms_key, context)
    sealed = seal_aes_ctr_legacy(
        key_service,
        secret,
        digest_method=digest,
    )

    secrets = dynamodb.Table(table)

    data = {
        'name': name,
        'version': paddedInt(version),
    }
    if comment:
        data['comment'] = comment
    data.update(sealed)

    return secrets.put_item(Item=data, ConditionExpression=Attr('name').not_exists()) 
Example #25
Source File: handler.py    From aws-tailor with GNU General Public License v3.0 4 votes vote down vote up
def handler(event, context):
    log.debug("Received event {}".format(json.dumps(event)))

    accountInfo = dynamodb.Table(os.environ['TAILOR_TABLENAME_ACCOUNTINFO'])

    try:
        print('context:resource-path', event['context']['resource-path'] == '/cloudabilty')
        print('body-json:accountId', re.match("^[0-9]{12}$", event['body-json']['accountId']))
    except Exception as e:
        print(e)
        print("regex not matching any values passed in request")
        raise Exception({"code": "4000", "message": "ERROR: Bad request"})

    # VPC DNS logic
    if event['context']['resource-path'] == '/cloudability' and \
            re.match("^[0-9]{12}$", event['body-json']['accountId']):

        requestId = str(uuid.uuid4())
        accountId = event['body-json']['accountId']
        stage = event['stage-variables']['stage']

        # Check if account already exists
        getAccountId = accountInfo.scan(
            ProjectionExpression='accountId, accountEmailAddress',
            FilterExpression=Attr('accountId').eq(accountId)
        )

        if getAccountId['Count'] == 0:
            print("Account not found")
            raise Exception({"code": "4040", "message": "ERROR: Not found"})

        elif int(getAccountId['Count']) > 0:

            # Update accountInfo with new requestId
            accountInfo.update_item(
                Key={
                    'accountEmailAddress': getAccountId['Items'][0]['accountEmailAddress']
                },
                UpdateExpression='SET #requestId = :val1',
                ExpressionAttributeNames={'#requestId': "requestId"},
                ExpressionAttributeValues={':val1': requestId}
            )

            # Build Lambda invoke payload
            message = {"requestId": requestId, "accountId": accountId, "accountEmailAddress": getAccountId['Items'][0]['accountEmailAddress'] }
            payload = {"Records": [{"Sns": {"Message": message}}]}

            # Call Lambda
            awslambda.invoke(
                FunctionName='talr-cloudability-' + stage,
                InvocationType='Event',
                Payload=json.dumps(payload),
            )

            return {"code": "2020", "message": "Request Accepted", "requestId": requestId}

    else:
        raise Exception({"code": "4000", "message": "ERROR: Bad request"}) 
Example #26
Source File: handler.py    From aws-tailor with GNU General Public License v3.0 4 votes vote down vote up
def handler(event, context):
    log.debug("Received event {}".format(json.dumps(event)))

    accountInfo = dynamodb.Table(os.environ['TAILOR_TABLENAME_ACCOUNTINFO'])

    try:
        print('context:resource-path', event['context']['resource-path'] == '/cloudtrail/{accountId}')
        print('path:accountId', re.match("^[0-9]{12}$", event['params']['path']['accountId']))
    except Exception as e:
        print(e)
        print("regex not matching any values passed in request")
        raise Exception({"code": "4000", "message": "ERROR: Bad request"})

    # Payload processing logic
    if event['context']['resource-path'] == '/cloudtrail/{accountId}' and \
            re.match("^[0-9]{12}$", event['params']['path']['accountId']):

        requestId = str(uuid.uuid4())
        accountId = event['params']['path']['accountId']
        stage = event['stage-variables']['stage']

        # Check if account is known to Tailor
        getAccountId = accountInfo.scan(
            ProjectionExpression='accountId, accountEmailAddress',
            FilterExpression=Attr('accountId').eq(accountId)
        )

        if getAccountId['Count'] == 0:
            print("Account not found")
            raise Exception({"code": "4040", "message": "ERROR: Not found"})

        elif int(getAccountId['Count']) > 0:

            # Update accountInfo with new requestId
            accountInfo.update_item(
                Key={
                    'accountEmailAddress': getAccountId['Items'][0]['accountEmailAddress']
                },
                UpdateExpression='SET #requestId = :val1',
                ExpressionAttributeNames={'#requestId': "requestId"},
                ExpressionAttributeValues={':val1': requestId}
            )

            # Build Lambda invoke payload
            message = {"requestId": requestId,
                       "accountId": accountId,
                       "accountEmailAddress": getAccountId['Items'][0]['accountEmailAddress']}
            payload = {"message": message}

            # Call Lambda
            awslambda.invoke(
                FunctionName='talr-cloudtrail-' + stage,
                InvocationType='Event',
                Payload=json.dumps(payload),
            )

            return {"code": "2020", "message": "Request Accepted", "requestId": requestId}

    else:
        raise Exception({"code": "4000", "message": "ERROR: Bad request"}) 
Example #27
Source File: dynamodb.py    From boto3-examples with MIT License 4 votes vote down vote up
def scan_item(
        self, table_name, attr1, attr2,
        total_items=None, start_key=None,
        table=None
    ):
        """
        Scan for an item with two attributes

        NOTE: SCAN OPERATION SCANS THE WHOLE TABLE AND TAKES CONSIDERABLE
        AMOUNT OF TIME, CONSUMES HIGH READ THROUGHPUT.
        AVOID USING THIS AS MUCH AS YOU CAN.
        TRY CREATING INDEX AND USE QUERY IF POSSIBLE

        PARAMS:
        @table_name: name of the table
        @attr1: Dict containing key and val of first attribute
        e.g. {'name': 'uuid', 'value': '077f4450-96ee-4ba8-8faa-831f6350a860'}
        @attr2: Dict containing key and val of second attribute
        e.g. {'name': 'date', 'value': '2017-02-12'}
        """
        if not table:
            dynamodb = self.conn
            table = dynamodb.Table(table_name)
        a1 = attr1['name']
        a1v = attr1['value']
        a2 = attr2['name']
        a2v = attr2['value']
        if not start_key:
            response = table.scan_item(
                FilterExpression=Attr(a1).eq(a1v) &
                Attr(a2).eq(a2v)
            )
        else:
            response = table.scan_item(
                FilterExpression=Attr(a1).eq(a1v) &
                Attr(a2).eq(a2v),
                ExclusiveStartKey=start_key
            )
        if not total_items:
            total_items = response['Items']
        else:
            total_items.extend(response['Items'])
        if response.get('LastEvaluatedKey'):
            start_key = response['LastEvaluatedKey']
            return_items = self.query_item(
                table_name=table_name, attr1=attr1,
                attr2=attr2, total_items=total_items,
                start_key=start_key, table=table
            )
            return return_items
        else:
            return total_items 
Example #28
Source File: scheduler_task_cleanup_action.py    From aws-ops-automator with Apache License 2.0 4 votes vote down vote up
def execute(self):

        self._logger_.info("{}, version {}", str(self.__class__).split(".")[-2], self.properties[ACTION_VERSION])
        self._logger_.debug("Implementation {}", __name__)

        self._logger_.info("Cleanup table {}", self.task_table)

        scanned_count = 0
        deleted_count = 0

        # calculate moment from when entries can be deleted
        dt = (self._datetime_.utcnow() - datetime(1970, 1, 1)).total_seconds()
        delete_before = int(dt - self.task_retention_seconds)
        self._logger_.info(INF_DELETE, datetime.fromtimestamp(delete_before).isoformat())

        #  status of deleted items for scan expression
        delete_status = [handlers.STATUS_COMPLETED]

        if not self.retain_failed_tasks:
            delete_status.append(handlers.STATUS_FAILED)
            delete_status.append(handlers.STATUS_TIMED_OUT)

        table = self._session_.resource("dynamodb").Table(self.task_table)
        add_retry_methods_to_resource(table, ["scan"], context=self._context_)

        args = {
            "Select": "ALL_ATTRIBUTES",
            "FilterExpression": (Attr(handlers.TASK_TR_STATUS).is_in(delete_status))
        }

        self._logger_.debug("table.scan arguments {}", args)

        # scan for items to delete
        while True:

            if self.time_out():
                break

            resp = table.scan_with_retries(**args)

            self._logger_.debug("table.scan result {}", resp)

            scanned_count += resp["ScannedCount"]

            to_delete = [i[handlers.TASK_TR_ID] for i in resp.get("Items", []) if
                         int(i[handlers.TASK_TR_CREATED_TS]) < delete_before]

            deleted_count += self.delete_tasks(to_delete)

            if "LastEvaluatedKey" in resp:
                args["ExclusiveStartKey"] = resp["LastEvaluatedKey"]
            else:
                break

        return {
            "items-scanned": scanned_count,
            "items-deleted": deleted_count
        }