Python boto3.dynamodb.conditions.Key() Examples

The following are code examples for showing how to use boto3.dynamodb.conditions.Key(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: cis   Author: mozilla-iam   File: user.py    Mozilla Public License 2.0 6 votes vote down vote up
def _update_with_transaction(self, user_profile):
        cis_profile_user_object = User(user_structure_json=json.loads(user_profile["profile"]))
        transact_items = {
            "Update": {
                "Key": {"id": {"S": user_profile["id"]}},
                "ExpressionAttributeValues": {
                    ":p": {"S": user_profile["profile"]},
                    ":u": {"S": user_profile["user_uuid"]},
                    ":pe": {"S": user_profile["primary_email"]},
                    ":pn": {"S": user_profile["primary_username"]},
                    ":sn": {"S": user_profile["sequence_number"]},
                    ":a": {"BOOL": json.loads(user_profile["profile"])["active"]["value"]},
                    ":fp": {"M": cis_profile_user_object.as_dynamo_flat_dict()},
                },
                "ConditionExpression": "attribute_exists(id)",
                "UpdateExpression": "SET profile = :p, primary_email = :pe, sequence_number = :sn, user_uuid = :u,"
                "primary_username = :pn, active = :a, flat_profile = :fp",
                "TableName": self.table.name,
                "ReturnValuesOnConditionCheckFailure": "NONE",
            }
        }
        return self._run_transaction([transact_items]) 
Example 2
Project: chaos   Author: th3sys   File: ig_executor.py    MIT License 6 votes vote down vote up
def GetSecurities(self, securities):
        try:
            self.__logger.info('Calling securities query ...')
            pairs = list(map(lambda x: Key('Symbol').eq(x[0]) & Key('Broker').eq(x[1]), securities))
            keyCondition = reduce(lambda x, y: x | y, pairs) if len(pairs) > 1 else pairs[0]

            with async_timeout.timeout(self.__timeout):
                response = await self.__loop.run_in_executor(None,
                                                             functools.partial(self.__Securities.scan,
                                                                               FilterExpression=keyCondition))
                return response['Items']

        except ClientError as e:
            self.__logger.error(e.response['Error']['Message'])
            return None
        except Exception as e:
            self.__logger.error(e)
            return None 
Example 3
Project: MPC   Author: jrknox1977   File: tweet.py    GNU General Public License v3.0 6 votes vote down vote up
def scan_table_allpages(self, table_name, filter_key=None, filter_value=None):
        """
        Perform a scan operation on table. Can specify filter_key (col name) and its value to be filtered.
        This gets all pages of results.
        Returns list of items.
        """
        table = self.dynamodb_resource.Table(table_name)

        if filter_key and filter_value:
            filtering_exp = Key(filter_key).eq(filter_value)
            response = table.scan(FilterExpression=filtering_exp)
        else:
            response = table.scan()

        items = response['Items']
        while True:
            print(len(response['Items']))
            if response.get('LastEvaluatedKey'):
                response = table.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
                items += response['Items']
            else:
                break
        return items 
Example 4
Project: MPC   Author: jrknox1977   File: rip_reddit.py    GNU General Public License v3.0 6 votes vote down vote up
def scan_table_allpages(self, table_name, filter_key=None, filter_value=None):
        """
        Perform a scan operation on table. Can specify filter_key (col name) and its value to be filtered.
        This gets all pages of results.
        Returns list of items.
        """
        table = self.dynamodb_resource.Table(table_name)

        if filter_key and filter_value:
            filtering_exp = Key(filter_key).eq(filter_value)
            response = table.scan(FilterExpression=filtering_exp)
        else:
            response = table.scan()

        items = response['Items']
        while True:
            print(len(response['Items']))
            if response.get('LastEvaluatedKey'):
                response = table.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
                items += response['Items']
            else:
                break
        return items 
Example 5
Project: sam-local-python-hot-reloading   Author: heitorlessa   File: users.py    MIT License 6 votes vote down vote up
def get_user(event):
    """ Quick Get Item with no paginators """
    data = json.loads(event["body"])
    userId = data.get('id', None)

    if not this_exist_not_null(userId):
        return respond("Invalid parameters", 400)

    try:
        ret = table.query(
            KeyConditionExpression=Key('id').eq(userId)
        )
        return respond(data=ret["Items"], status=200)
    except ClientError as e:
        print(e)
        return respond(data="Operation failed", status=500) 
Example 6
Project: sam-local-python-hot-reloading   Author: heitorlessa   File: users.py    MIT License 6 votes vote down vote up
def delete_user(event):
    """ Quick Delete Item """
    data = json.loads(event["body"])
    userId = data.get('id', None)

    if not this_exist_not_null(userId):
        return respond("Invalid parameters", 400)

    try:
        ret = table.delete_item(
            Key={
                "id": userId
            }
        )

        if aws_request_was_successful(ret):
            return respond(f"User {userId} Deleted successfully.", 200)
    except ClientError as e:
        print(e)
        return respond(data="Operation failed", status=500) 
Example 7
Project: Building-Serverless-Microservices-in-Python   Author: PacktPublishing   File: dynamo_query_table.py    MIT License 6 votes vote down vote up
def query_dynamo_record_by_parition_sort_key(self, partition_key, partition_value,
                                                 sort_key, sort_value):
        try:
            response = self.table.query(
                KeyConditionExpression=Key(partition_key).eq(partition_value)
                                       & Key(sort_key).gte(sort_value))
            for record in response.get('Items'):
                print(json.dumps(record, cls=DecimalEncoder))
            return

        except Exception as e:
            print('Exception %s type' % str(type(e)))
            print('Exception message: %s ' % str(e)) 
Example 8
Project: aws-media-services-vod-automation   Author: aws-samples   File: app.py    Apache License 2.0 6 votes vote down vote up
def job_progress(jobId):
    DYNAMO_TABLE_NAME = os.environ['JobTable']
    DYNAMO_INDEX_NAME = 'id-createdAt-index' 

    logger.info('GET progress for {} from {}'.format(jobId, DYNAMO_TABLE_NAME))
    
    try:
        table = dynamodb_resource.Table(DYNAMO_TABLE_NAME)
        
        response = table.query(IndexName=DYNAMO_INDEX_NAME, KeyConditionExpression=Key('id').eq(jobId))
    
    except ClientError as e:
        logger.info("ClientError from Dynamodb {}".format(e.response['Error']['Message']))
        raise BadRequestError("Dynamodb returned error message '%s'" % e.response['Error']['Message'])
    except Exception as e:
        logger.info("ClientError from Dynamodb {}".format(e.response['Error']['Message']))
        raise ChaliceViewError("Dynamodb returned error message '%s'" % e.response['Error']['Message'])

    print (json.dumps(response, cls=DecimalEncoder))
    # there is no information about this job yet
    if response['Count'] > 0:
        return response['Items'][0]
    else:
        return {} 
Example 9
Project: aws-media-services-vod-automation   Author: aws-samples   File: event_collector.py    Apache License 2.0 6 votes vote down vote up
def getMediaConvertJob(id, JOBTABLE):
    """
    Retrieve a job data structure from dynamodb
    :param id:  id of the job
    :param JOBTABLE: Name of the dynamodb job table for this stack
    """    
    try:
        table = DYNAMO_CLIENT.Table(JOBTABLE)
        response = table.get_item(Key={'id': id}, ConsistentRead=True)
    
    except ClientError as e:
        print(e.response['Error']['Message'])
    else:
        if 'Item' not in response:
            return None
        else:
            item = response['Item']
            print("GetItem succeeded:")
            #print(json.dumps(item, indent=4, cls=DecimalEncoder))
            return item 
Example 10
Project: kev   Author: capless   File: db.py    GNU General Public License v3.0 6 votes vote down vote up
def parse_filters(self, filters, doc_class):
        index_name = None
        filter_expression_list = []
        query_params = {}
        for idx, filter in enumerate(filters):
            prop_name, prop_value = filter.split(':')[3:5]
            if idx == 0:
                prop = doc_class()._base_properties[prop_name]
                index_name = prop.kwargs.get(self.index_field_name, None) or \
                             self.default_index_name.format(prop_name)
                query_params['KeyConditionExpression'] = Key(prop_name).eq(prop_value)
            else:
                filter_expression_list.append(Attr(prop_name).eq(prop_value))
        if len(filter_expression_list) > 1:
            query_params['FilterExpression'] = And(*filter_expression_list)
        elif len(filter_expression_list) == 1:
            query_params['FilterExpression'] = filter_expression_list[0]
        if index_name != '_id':
            query_params['IndexName'] = index_name
        return query_params 
Example 11
Project: aws_lambda_for_python_example   Author: crazyfan5566   File: my12_boto3_working_with_dynamodb.py    MIT License 6 votes vote down vote up
def dynamodb_query(table):
    '''
    依條件查詢
    http://boto3.readthedocs.io/en/latest/guide/dynamodb.html#querying-and-scanning

    注意要import這行:
    from boto3.dynamodb.conditions import Key, Attr
    '''

    response = table.query(
        KeyConditionExpression=Key('Artist').eq('五月天'),
        ScanIndexForward=True,  # sort
        Limit=10
    )

    print(response['Items'])  # dump
    print(response['Items'][0])  # first record

    return None 
Example 12
Project: aws_lambda_for_python_example   Author: crazyfan5566   File: my12_boto3_working_with_dynamodb.py    MIT License 6 votes vote down vote up
def dynamodb_update_item(table):
    '''
    Updating Item
    '''
    s_artist = '五月天'
    s_title = '傷心的人別聽慢歌'

    table.update_item(
        Key={
            'Artist':  s_artist,
            'SongTitle': s_title,
        },
        UpdateExpression='SET view_cnt = :val_1',
        ExpressionAttributeValues={
            ':val_1': 100
        }
    )

    return None 
Example 13
Project: aws_lambda_for_python_example   Author: crazyfan5566   File: my12_boto3_working_with_dynamodb.py    MIT License 6 votes vote down vote up
def dynamodb_get_item(table):
    '''
    Getting an Item
    '''

    s_artist = '五月天'
    s_title = '傷心的人別聽慢歌'

    response = table.get_item(
        Key={
            'Artist':  s_artist,
            'SongTitle': s_title,
        }
    )
    item = response['Item']
    print(item)

    return None 
Example 14
Project: aws_lambda_for_python_example   Author: crazyfan5566   File: my12_boto3_working_with_dynamodb.py    MIT License 6 votes vote down vote up
def dynamodb_delete_item(table):
    '''
    Deleting Item
    '''

    s_artist = '五月天'
    s_title = '傷心的人別聽慢歌'

    table.delete_item(
        Key={
            'Artist':  s_artist,
            'SongTitle': s_title,
        }
    )

    return None 
Example 15
Project: polly-recorder   Author: soofaloofa   File: app.py    MIT License 6 votes vote down vote up
def upload_to_s3(record_id, s3_bucket):
    """
    Upload the tmp file to S3.

    Returns the S3 URL of the uploaded result.
    """
    s3.upload_file('/tmp/' + record_id,
                   s3_bucket,
                   record_id + ".mp3")

    s3.put_object_acl(ACL='public-read',
                      Bucket=s3_bucket,
                      Key=record_id + ".mp3")

    location = s3.get_bucket_location(Bucket=s3_bucket)
    region = location['LocationConstraint']

    if region is None:
        url_begining = "https://s3.amazonaws.com/"
    else:
        url_begining = "https://s3-" + str(region) + ".amazonaws.com/" \

    url = url_begining + s3_bucket + "/" + record_id + ".mp3"

    return url 
Example 16
Project: polly-recorder   Author: soofaloofa   File: app.py    MIT License 6 votes vote down vote up
def get_recording(record_id):
    """
    Get existing recordings.
    """
    if record_id == "*":
        # List all recordings
        dynamodb = boto3.resource('dynamodb')
        table = dynamodb.Table(DYNAMO_DB_TABLE)
        items = table.scan()

        return items["Items"]
    else:
        dynamodb = boto3.resource('dynamodb')
        table = dynamodb.Table(DYNAMO_DB_TABLE)

        items = table.query(
            KeyConditionExpression=Key('id').eq(record_id)
        )

        return items["Items"] 
Example 17
Project: streamalert   Author: airbnb   File: alert_table.py    Apache License 2.0 6 votes vote down vote up
def get_alert_record(self, rule_name, alert_id):
        """Get a single alert record from the alerts table.

        Args:
            rule_name (str): Name of the rule the alert triggered on
            alert_id (str): Alert UUID

        Returns:
            (dict): Dynamo record corresponding to this alert, or None if the alert was not found.
        """
        kwargs = {
            'ConsistentRead': True,
            'KeyConditionExpression': Key('RuleName').eq(rule_name) & Key('AlertID').eq(alert_id)
        }
        items = list(self._paginate(self._table.query, kwargs))
        return items[0] if items else None

    # ---------- Add/Delete/Update Operations ---------- 
Example 18
Project: streamalert   Author: airbnb   File: alert_table.py    Apache License 2.0 6 votes vote down vote up
def mark_as_dispatched(self, alert):
        """Mark a specific alert as dispatched (in progress).

        Args:
            alert (Alert): Alert instance which has just been sent to the alert processor
        """
        # Update the alerts table with the dispatch time, but only if the alert still exists.
        # (The alert processor could have deleted the alert before this table update finishes).
        self._table.update_item(
            Key=alert.dynamo_key,
            UpdateExpression='SET Attempts = :attempts, Dispatched = :dispatched',
            ExpressionAttributeValues={
                ':attempts': alert.attempts,
                ':dispatched': alert.dispatched.strftime(Alert.DATETIME_FORMAT)
            },
            ConditionExpression='attribute_exists(AlertID)'
        ) 
Example 19
Project: cloudtrail-tracker   Author: grycap   File: Querys.py    Apache License 2.0 6 votes vote down vote up
def users_list():
    """Return a list of users"""
    listName = 'listUsers'
    pe = Key('eventID').eq('1') #what we want to search

    table = dynamodb_resource.Table(table_name)

    response = table.query(
        KeyConditionExpression=pe,
    )
    data = response['Items']
    users = list(data[0][listName].keys())
    #search_in_events(users,data,users_itemName)
    # print(users)
    while 'LastEvaluatedKey' in response:
        response = table.query(
            ExclusiveStartKey=response['LastEvaluatedKey'],
            ProjectionExpression=pe,
        )
        data= response['Items']
        users.append(list(data[0][listName].keys()))
        #search_in_events(users, data, users_itemName)
        # print(users)

    return users 
Example 20
Project: cloudtrail-tracker   Author: grycap   File: Querys.py    Apache License 2.0 6 votes vote down vote up
def services_list():
    """Return a list of services"""
    listName = 'services'
    pe = Key('eventID').eq('1') #what we want to search

    table = dynamodb_resource.Table(table_name)

    response = table.query(
        KeyConditionExpression=pe,
    )
    data = response['Items']
    services = list(data[0][listName].keys())
    #search_in_events(users,data,users_itemName)
    # print(users)
    while 'LastEvaluatedKey' in response:
        response = table.query(
            ExclusiveStartKey=response['LastEvaluatedKey'],
            ProjectionExpression=pe,
        )
        data= response['Items']
        services.append(list(data[0][listName].keys()))
        #search_in_events(users, data, users_itemName)
        # print(users)

    return services 
Example 21
Project: cloudtrail-tracker   Author: grycap   File: Querys.py    Apache License 2.0 6 votes vote down vote up
def parameters_list():
    """Return a list of parameters to query"""
    listName = 'cols'
    pe = Key('eventID').eq('1') #what we want to search

    table = dynamodb_resource.Table(table_name)

    response = table.query(
        KeyConditionExpression=pe,
    )
    data = response['Items']
    parameters = list(data[0][listName].keys())
    #search_in_events(users,data,users_itemName)
    # print(users)
    while 'LastEvaluatedKey' in response:
        response = table.query(
            ExclusiveStartKey=response['LastEvaluatedKey'],
            ProjectionExpression=pe,
        )
        data= response['Items']
        parameters.append(list(data[0][listName].keys()))
        #search_in_events(users, data, users_itemName)
        # print(users)

    return parameters 
Example 22
Project: cis   Author: mozilla-iam   File: user.py    Mozilla Public License 2.0 5 votes vote down vote up
def _delete_without_transaction(self, user_profile):
        return self.table.delete_item(Key={"id": user_profile["id"]}) 
Example 23
Project: cis   Author: mozilla-iam   File: user.py    Mozilla Public License 2.0 5 votes vote down vote up
def _update_batch_with_transaction(self, list_of_profiles):
        transact_items = []
        for user_profile in list_of_profiles:
            cis_profile_user_object = User(user_structure_json=json.loads(user_profile["profile"]))
            update_expression = "SET profile = :p, primary_email = :pe, \
                sequence_number = :sn, user_uuid = :u, primary_username = :pn, \
                active = :a, flat_profile = :fp"
            transact_item = {
                "Update": {
                    "Key": {"id": {"S": user_profile["id"]}},
                    "ExpressionAttributeValues": {
                        ":p": {"S": user_profile["profile"]},
                        ":u": {"S": user_profile["user_uuid"]},
                        ":pe": {"S": user_profile["primary_email"]},
                        ":pn": {"S": user_profile["primary_username"]},
                        ":sn": {"S": user_profile["sequence_number"]},
                        ":a": {"BOOL": json.loads(user_profile["profile"])["active"]["value"]},
                        ":fp": {"M": cis_profile_user_object.as_dynamo_flat_dict()},
                    },
                    "ConditionExpression": "attribute_exists(id)",
                    "UpdateExpression": update_expression,
                    "TableName": self.table.name,
                    "ReturnValuesOnConditionCheckFailure": "NONE",
                }
            }
            transact_items.append(transact_item)
        logger.debug("Attempting to update batch of transactions for: {}".format(transact_items))
        return self._run_transaction(transact_items) 
Example 24
Project: cis   Author: mozilla-iam   File: user.py    Mozilla Public License 2.0 5 votes vote down vote up
def find_by_id(self, id):
        result = self.table.query(KeyConditionExpression=Key("id").eq(id))
        return result 
Example 25
Project: cis   Author: mozilla-iam   File: user.py    Mozilla Public License 2.0 5 votes vote down vote up
def find_by_email(self, primary_email):
        result = self.table.query(
            IndexName="{}-primary_email".format(self.table.table_name),
            KeyConditionExpression=Key("primary_email").eq(primary_email),
        )
        return result 
Example 26
Project: cis   Author: mozilla-iam   File: user.py    Mozilla Public License 2.0 5 votes vote down vote up
def find_by_username(self, primary_username):
        result = self.table.query(
            IndexName="{}-primary_username".format(self.table.table_name),
            KeyConditionExpression=Key("primary_username").eq(primary_username),
        )
        return result 
Example 27
Project: serverless-instagram   Author: jjanczyszyn   File: list_images.py    MIT License 5 votes vote down vote up
def handler(event, context):
    logger.info('Received event: {}'.format(event))
    user = event['requestContext']['authorizer']['principalId']

    table = DYNAMO_DB.Table(IMAGE_INFO_TABLE_NAME)
    filtering_exp = Key('user').eq(user)
    result = table.query(KeyConditionExpression=filtering_exp, ScanIndexForward=False)

    return build_response(200, result['Items']) 
Example 28
Project: chaos   Author: th3sys   File: vix_roll_trader.py    MIT License 5 votes vote down vote up
def GetSecurities(self):
        try:
            self.Logger.info('Calling securities query ...')
            response = self.__Securities.query(
                KeyConditionExpression=Key('Symbol').eq('VX') & Key('Broker').eq('IG'))
        except ClientError as e:
            self.Logger.error(e.response['Error']['Message'])
            return None
        except Exception as e:
            self.Logger.error(e)
            return None
        else:
            if 'Items' in response:
                return response['Items'] 
Example 29
Project: chaos   Author: th3sys   File: vix_roll_trader.py    MIT License 5 votes vote down vote up
def GetQuotes(self, symbol, date):
        try:
            self.Logger.info('Calling quotes query Date key: %s' % date)
            response = self.__QuotesEod.query(
                KeyConditionExpression=Key('Symbol').eq(symbol) & Key('Date').eq(date)
            )
        except ClientError as e:
            self.Logger.error(e.response['Error']['Message'])
            return None
        except Exception as e:
            self.Logger.error(e)
            return None
        else:
            if 'Items' in response:
                return response['Items'] 
Example 30
Project: chaos   Author: th3sys   File: ig_executor.py    MIT License 5 votes vote down vote up
def __init__(self):
        self.Url = ''
        self.Key = ''
        self.Identifier = ''
        self.Password = ''
        self.EAddress = ''
        self.EUser = ''
        self.EPassword = ''
        self.ESmtp = '' 
Example 31
Project: dynamoplus   Author: antessio   File: repositories.py    GNU General Public License v3.0 5 votes vote down vote up
def get(self, id: str):
        # TODO: copy from query -> if the indexKeys is empty then get by primary key, otherwise get by global secondary index
        # it means if needed first get from index, then by primary key or, in case of index it throws a non supported operation exception
        model = self.getModelFromDocument({self.collection.id_key: id})
        result = self.table.get_item(
            Key={
                'pk': model.pk(),
                'sk': model.sk()
            })

        return Model.from_dynamo_db_item(result[u'Item'], self.collection) if 'Item' in result else None 
Example 32
Project: dynamoplus   Author: antessio   File: repositories.py    GNU General Public License v3.0 5 votes vote down vote up
def update(self, document: dict):
        model = self.getModelFromDocument(document)
        dynamo_db_item = model.to_dynamo_db_item()
        if dynamo_db_item.keys():
            # only updates attributes in the id_key or pk or sk
            logger.info("updating {} ".format(dynamo_db_item))

            update_expression = "SET " + ", ".join(map(lambda k: "#{}=:{}".format(k,k), filter(
                lambda k: k != self.collection.id_key and k != "pk" and k != "sk",
                dynamo_db_item.keys())))
            expression_value = dict(
                map(lambda kv: (":{}".format(kv[0]), kv[1]),
                    filter(
                        lambda kv: kv[0] != self.collection.id_key and kv[0] != "pk" and kv[0] != "sk", dynamo_db_item.items())))
            expression_attribute_names = {}
            for k in dynamo_db_item.keys():
                if k != self.collection.id_key and k != "pk" and k != "sk":
                    expression_attribute_names["#{}".format(k)]=k
            response = self.table.update_item(
                Key={
                    'pk': model.pk(),
                    'sk': model.sk()
                },
                UpdateExpression=update_expression,
                ExpressionAttributeValues=expression_value,
                ExpressionAttributeNames=expression_attribute_names,
                ReturnValues="UPDATED_NEW"
            )
            logger.info("Response from update operation is " + response.__str__())
            if response['ResponseMetadata']['HTTPStatusCode'] == 200:
                return model
            else:
                logger.error("The status is {}".format(response['ResponseMetadata']['HTTPStatusCode']))
                return None
        else:
            raise Exception("dynamo db item empty ") 
Example 33
Project: dynamoplus   Author: antessio   File: repositories.py    GNU General Public License v3.0 5 votes vote down vote up
def delete(self, id: str):
        model = self.getModelFromDocument({self.collection.id_key: id})
        response = self.table.delete_item(
            Key={
                'pk': model.pk(),
                'sk': model.sk()
            })
        if response['ResponseMetadata']['HTTPStatusCode'] != 200:
            logger.error("The status is {}".format(response['ResponseMetadata']['HTTPStatusCode']))
            raise Exception("Error code {}".format(response['ResponseMetadata']['HTTPStatusCode'])) 
Example 34
Project: dynamoplus   Author: antessio   File: repositories.py    GNU General Public License v3.0 5 votes vote down vote up
def find(self, query: Query):
        logger.info(" Received query={}".format(query.__str__()))
        document = query.document
        index_model = IndexModel(self.collection, document, query.index)
        ordering_key = query.index.ordering_key if query.index else None
        logger.info("order by is {} ".format(ordering_key))
        limit = query.limit
        start_from = query.start_from
        if index_model.data() is not None:
            if ordering_key is None:
                key = Key('sk').eq(index_model.sk()) & Key('data').eq(index_model.data())
                logger.info(
                    "The key that will be used is sk={} is equal data={}".format(index_model.sk(), index_model.data()))
            else:
                key = Key('sk').eq(index_model.sk()) & Key('data').begins_with(index_model.data())
                logger.info(
                    "The key that will be used is sk={} begins with data={}".format(index_model.sk(),
                                                                                    index_model.data()))
        else:
            key = Key('sk').eq(index_model.sk())
            logger.info("The key that will be used is sk={} with no data".format(index_model.sk()))

        dynamo_query = dict(
            IndexName="sk-data-index",
            KeyConditionExpression=key,
            Limit=limit,
            ExclusiveStartKey=start_from
        )
        response = self.table.query(
            **{k: v for k, v in dynamo_query.items() if v is not None}
        )
        logger.info("Response from dynamo db {}".format(str(response)))
        last_key = None

        if 'LastEvaluatedKey' in response:
            last_key = response['LastEvaluatedKey']
        return QueryResult(list(map(lambda i: Model.from_dynamo_db_item(i, self.collection), response[u'Items'])), last_key) 
Example 35
Project: MPC   Author: jrknox1977   File: tweet.py    GNU General Public License v3.0 5 votes vote down vote up
def read_table_item(self, table_name, pk_name, pk_value):
        """
        Return item read by primary key.
        """
        table = self.dynamodb_resource.Table(table_name)
        response = table.get_item(Key={pk_name: pk_value})
        return response 
Example 36
Project: MPC   Author: jrknox1977   File: tweet.py    GNU General Public License v3.0 5 votes vote down vote up
def delete_item(self, table_name, pk_name, pk_value):
        """
        Delete an item (row) in table from its primary key.
        """
        table = self.dynamodb_resource.Table(table_name)
        response = table.delete_item(Key={pk_name: pk_value})
        return response 
Example 37
Project: MPC   Author: jrknox1977   File: tweet.py    GNU General Public License v3.0 5 votes vote down vote up
def scan_table_firstpage(self, table_name, filter_key=None, filter_value=None):
        """
        Perform a scan operation on table. Can specify filter_key (col name) and its value to be filtered.
        This gets only first page of results in pagination. Returns the response.
        """
        table = self.dynamodb_resource.Table(table_name)

        if filter_key and filter_value:
            filtering_exp = Key(filter_key).eq(filter_value)
            response = table.scan(FilterExpression=filtering_exp)
        else:
            response = table.scan()
        return response 
Example 38
Project: MPC   Author: jrknox1977   File: tweet.py    GNU General Public License v3.0 5 votes vote down vote up
def query_table(self, table_name, filter_key=None, filter_value=None):
        """
        Perform a query operation on the table. Can specify filter_key (col name) and its value to be filtered.
        Returns the response.
        """
        table = self.dynamodb_resource.Table(table_name)

        if filter_key and filter_value:
            filtering_exp = Key(filter_key).eq(filter_value)
            response = table.query(KeyConditionExpression=filtering_exp)
        else:
            response = table.query()
        return response 
Example 39
Project: MPC   Author: jrknox1977   File: rip_reddit.py    GNU General Public License v3.0 5 votes vote down vote up
def read_table_item(self, table_name, pk_name, pk_value):
        """
        Return item read by primary key.
        """
        table = self.dynamodb_resource.Table(table_name)
        response = table.get_item(Key={pk_name: pk_value})
        return response 
Example 40
Project: MPC   Author: jrknox1977   File: rip_reddit.py    GNU General Public License v3.0 5 votes vote down vote up
def delete_item(self, table_name, pk_name, pk_value):
        """
        Delete an item (row) in table from its primary key.
        """
        table = self.dynamodb_resource.Table(table_name)
        response = table.delete_item(Key={pk_name: pk_value})
        return response 
Example 41
Project: MPC   Author: jrknox1977   File: rip_reddit.py    GNU General Public License v3.0 5 votes vote down vote up
def scan_table_firstpage(self, table_name, filter_key=None, filter_value=None):
        """
        Perform a scan operation on table. Can specify filter_key (col name) and its value to be filtered.
        This gets only first page of results in pagination. Returns the response.
        """
        table = self.dynamodb_resource.Table(table_name)

        if filter_key and filter_value:
            filtering_exp = Key(filter_key).eq(filter_value)
            response = table.scan(FilterExpression=filtering_exp)
        else:
            response = table.scan()
        return response 
Example 42
Project: MPC   Author: jrknox1977   File: rip_reddit.py    GNU General Public License v3.0 5 votes vote down vote up
def query_table(self, table_name, filter_key=None, filter_value=None):
        """
        Perform a query operation on the table. Can specify filter_key (col name) and its value to be filtered.
        Returns the response.
        """
        table = self.dynamodb_resource.Table(table_name)

        if filter_key and filter_value:
            filtering_exp = Key(filter_key).eq(filter_value)
            response = table.query(KeyConditionExpression=filtering_exp)
        else:
            response = table.query()
        return response 
Example 43
Project: sam-local-python-hot-reloading   Author: heitorlessa   File: users.py    MIT License 5 votes vote down vote up
def update_user(event):
    """ Simple Single Update """

    data = json.loads(event["body"])
    userId = data.get('id', None)
    userEmail = data.get('email', None)

    if (
           not this_exist_not_null(userId) or
           not this_exist_not_null(userEmail)
       ):
        return respond("Invalid parameters", 400)

    try:
        ret = table.update_item(
            Key={
                "id": userId
            },
            UpdateExpression='SET email = :newvalue',
            ExpressionAttributeValues={
                ':newvalue': userEmail
            }
        )
        if aws_request_was_successful(ret):
            return respond(f"User {userId} Updated successfully.", 200)
    except ClientError as e:
        print(e)
        return respond("Operation failed", 500) 
Example 44
Project: sam-local-python-hot-reloading   Author: heitorlessa   File: bootstrap_dynamodb.py    MIT License 5 votes vote down vote up
def bootstrap_table(table_name):
    """Adds basic data

    Ideally this would be a group command to accept params or a file

    """

    userId = str(uuid.uuid4())
    userEmail = fake.email()

    try:
        table = dynamodb.Table(table_name)
        params = {
            "Item": {
                "id": userId,
                "email": userEmail
            }
        }
        table.put_item(**params)        # This format will accept duplicates of same email
        response = table.query(
            KeyConditionExpression=Key('id').eq(userId)
        )
        item = response['Items'][0]
        print(item)
        print("[+] Bootstrapping ran successfully...")
    except ClientError as error_message:
        print("[!] Failed to bootstrap table with demo data...")
        print(error_message) 
Example 45
Project: Building-Serverless-Microservices-in-Python   Author: PacktPublishing   File: dynamo_query_table.py    MIT License 5 votes vote down vote up
def query_dynamo_record_by_parition(self, parition_key, parition_value):
        try:
            response = self.table.query(
                KeyConditionExpression=Key(parition_key).eq(parition_value))
            for record in response.get('Items'):
                print(json.dumps(record, cls=DecimalEncoder))
            return

        except Exception as e:
            print('Exception %s type' % str(type(e)))
            print('Exception message: %s ' % str(e)) 
Example 46
Project: Building-Serverless-Microservices-in-Python   Author: PacktPublishing   File: lambda_return_dynamo_records.py    MIT License 5 votes vote down vote up
def query_by_partition_and_sort_key(self, partition_key, partition_value,
                                        sort_key, sort_value):
        response = self.db_table.query(KeyConditionExpression=
                                       Key(partition_key).eq(partition_value)
                                       & Key(sort_key).gte(sort_value))

        return response.get('Items') 
Example 47
Project: Building-Serverless-Microservices-in-Python   Author: PacktPublishing   File: lambda_return_dynamo_records.py    MIT License 5 votes vote down vote up
def query_by_partition_key(self, partition_key, partition_value):
        response = self.db_table.query(KeyConditionExpression=
                                       Key(partition_key).eq(partition_value))

        return response.get('Items') 
Example 48
Project: Building-Serverless-Microservices-in-Python   Author: PacktPublishing   File: dynamo_query_table.py    MIT License 5 votes vote down vote up
def query_dynamo_record_by_parition(self, parition_key, parition_value):
        try:
            response = self.table.query(
                KeyConditionExpression=Key(parition_key).eq(parition_value))
            for record in response.get('Items'):
                print(json.dumps(record, cls=DecimalEncoder))
            return

        except Exception as e:
            print('Exception %s type' % str(type(e)))
            print('Exception message: %s ' % str(e)) 
Example 49
Project: Building-Serverless-Microservices-in-Python   Author: PacktPublishing   File: lambda_return_dynamo_records.py    MIT License 5 votes vote down vote up
def query_by_partition_and_sort_key(self, partition_key, partition_value,
                                        sort_key, sort_value):
        response = self.db_table.query(KeyConditionExpression=
                                       Key(partition_key).eq(partition_value)
                                       & Key(sort_key).gte(sort_value))

        return response.get('Items') 
Example 50
Project: Building-Serverless-Microservices-in-Python   Author: PacktPublishing   File: lambda_return_dynamo_records.py    MIT License 5 votes vote down vote up
def query_by_partition_key(self, partition_key, partition_value):
        response = self.db_table.query(KeyConditionExpression=
                                       Key(partition_key).eq(partition_value))

        return response.get('Items') 
Example 51
Project: Building-Serverless-Microservices-in-Python   Author: PacktPublishing   File: dynamo_query_table.py    MIT License 5 votes vote down vote up
def query_dynamo_record_by_parition(self, parition_key, parition_value):
        try:
            response = self.table.query(
                KeyConditionExpression=Key(parition_key).eq(parition_value))
            for record in response.get('Items'):
                print(json.dumps(record, cls=DecimalEncoder))
            return

        except Exception as e:
            print('Exception %s type' % str(type(e)))
            print('Exception message: %s ' % str(e)) 
Example 52
Project: Building-Serverless-Microservices-in-Python   Author: PacktPublishing   File: dynamo_query_table.py    MIT License 5 votes vote down vote up
def query_dynamo_record_by_parition_sort_key(self, partition_key, partition_value,
                                                 sort_key, sort_value):
        try:
            response = self.table.query(
                KeyConditionExpression=Key(partition_key).eq(partition_value)
                                       & Key(sort_key).gte(sort_value))
            for record in response.get('Items'):
                print(json.dumps(record, cls=DecimalEncoder))
            return

        except Exception as e:
            print('Exception %s type' % str(type(e)))
            print('Exception message: %s ' % str(e)) 
Example 53
Project: Building-Serverless-Microservices-in-Python   Author: PacktPublishing   File: lambda_return_dynamo_records.py    MIT License 5 votes vote down vote up
def query_by_partition_and_sort_key(self, partition_key, partition_value,
                                        sort_key, sort_value):
        response = self.db_table.query(KeyConditionExpression=
                                       Key(partition_key).eq(partition_value)
                                       & Key(sort_key).gte(sort_value))

        return response.get('Items') 
Example 54
Project: aws-media-services-vod-automation   Author: aws-samples   File: event_collector.py    Apache License 2.0 5 votes vote down vote up
def get_signed_url(expires_in, bucket, obj):
    """
    Generate a signed URL for an object in S3 so it can be accessed as an HTTP resource
    :param expires_in:  URL Expiration time in seconds
    :param bucket:
    :param obj:         S3 Key name
    :return:            Signed URL
    """
    s3_cli = boto3.client("s3")
    presigned_url = s3_cli.generate_presigned_url('get_object', Params={'Bucket': bucket, 'Key': obj},
                                                  ExpiresIn=expires_in)
    return presigned_url 
Example 55
Project: aws-media-services-vod-automation   Author: aws-samples   File: event_collector.py    Apache License 2.0 5 votes vote down vote up
def jobMediaInfo(job):
    """
    run mediainfo analysis on the job inputs and add the results to the job JSON
    :param job:  JSON job data struture from dynamodb
    """
    
    # Loop through input videos in the job
    for input in job['settings']['inputs']:
        s3_path = input['fileInput']
        urlp = urlparse(s3_path)
        # Extract the Key and Bucket names for the inputs
        key = urlp[2]
        key = key.lstrip('/')
        bucket = urlp[1]
        
        signed_url = get_signed_url(SIGNED_URL_EXPIRATION, bucket, key)
        logger .info("Signed URL: {}".format(signed_url))
        
        print ("bucket and key "+bucket+" "+key)
        
        # Launch MediaInfo
        # Pass the signed URL of the uploaded asset to MediaInfo as an input
        # MediaInfo will extract the technical metadata from the asset
        # The extracted metadata will be outputted in XML format and
        # stored in the variable xml_output
        try:
            xml_output = subprocess.check_output(["./mediainfo", "--full", "--output=XML", signed_url])
            print(xml_output)
        except subprocess.CalledProcessError as e:
            print (e.output)
            
        json_output = xmltodict.parse(xml_output)
        
        input['mediainfo'] = json_output['Mediainfo']
        
        #print(json.dumps(json_output, indent=4, cls=DecimalEncoder))

    return True 
Example 56
Project: open-house-crawler   Author: data-skeptic   File: oh-get-parser-work.py    GNU General Public License v3.0 5 votes vote down vote up
def lambda_handler(event, context):
    work = {}
    response = table.query(KeyConditionExpression=Key('status-mask').eq(1),IndexName='status-mask-index')
    items = response['Items']
    i = int(random.random() * len(items))
    if len(items) > 0:
        item = items[i]
        work['item'] = item
        # Get crawl content 
        work['html'] = ''
    return work 
Example 57
Project: open-house-crawler   Author: data-skeptic   File: OH-review-crawled-page.py    GNU General Public License v3.0 5 votes vote down vote up
def lambda_handler(event, context):
    # TODO: can we randomize the result so simultaneous workers don't collide?
    # Maybe if it gets that frequent then we should be using SQS instead anyway.
    retry = 10
    response = table.query(
        IndexName='status-mask-index',
        KeyConditionExpression=Key('status-mask').eq(2), Limit=retry)
    #
    if response['Count'] == 0:
        s3key = 'empty.htm'
    else:
        ii = 0
        while ii < retry and ii < response['Count']:
            item = response['Items'][ii]
            url = item['uri']
            tld = tldextract_extract(url)
            if tld['subdomain'] != '' and tld['subdomain'] != 'www':
                tld = tld['subdomain'] + '.' + tld['domain'] + '.' + tld['suffix']
            else:
                tld = tld['domain'] + '.' + tld['suffix']
            i = url.find(tld)
            s3key = tld + url[i+len(tld):]
            print(s3key)
            try:
                obj = s3.Object(bucket, s3key)
                data = io.BytesIO()
                obj.download_fileobj(data)
                s = data.getvalue().decode("utf-8")
                resp = {"content": s, "url": url}
                return resp
            except:
                pass
            ii += 1
    s3key = 'empty.htm'
    obj = s3.Object(bucket, s3key)
    data = io.BytesIO()
    obj.download_fileobj(data)
    s = data.getvalue().decode("utf-8")
    resp = {"content": s, "url": "done"}
    return resp 
Example 58
Project: kev   Author: capless   File: db.py    GNU General Public License v3.0 5 votes vote down vote up
def delete(self, doc_obj):
        self._indexer.delete_item(Key={'_id': doc_obj._data['_id']}) 
Example 59
Project: kev   Author: capless   File: db.py    GNU General Public License v3.0 5 votes vote down vote up
def get(self, doc_obj, doc_id):
        response = self._indexer.get_item(Key={'_id': doc_obj.get_doc_id(doc_id)})
        doc = response.get('Item', None)
        if not doc:
            raise DocNotFoundError
        return doc_obj(**doc) 
Example 60
Project: kev   Author: capless   File: db.py    GNU General Public License v3.0 5 votes vote down vote up
def flush_db(self):
        obj_list = self._indexer.scan()['Items']
        for i in obj_list:
            self._indexer.delete_item(Key={'_id': i['_id']})

    # Indexing Methods 
Example 61
Project: makerlab-103-Polly   Author: ACloudGuru   File: getposts.py    GNU General Public License v3.0 5 votes vote down vote up
def lambda_handler(event, context):
    
    postId = event["postId"]
    
    dynamodb = boto3.resource('dynamodb')
    table = dynamodb.Table(os.environ['DB_TABLE_NAME'])
    
    if postId=="*":
        items = table.scan()
    else:
        items = table.query(
            KeyConditionExpression=Key('id').eq(postId)
        )
    
    return items["Items"] 
Example 62
Project: pinboard-backup   Author: QuinnyPig   File: link_importer.py    MIT License 5 votes vote down vote up
def prune_stale(issue):
    response = table.query(
        KeyConditionExpression=Key('issue').eq(issue),
        FilterExpression=Attr('timestamp').lt(int(time.time()) - 350)
    )
    for i in response['Items']:
        table.delete_item(
            Key={
                'issue': i['issue'],
                'url': i['url']
            }
        ) 
Example 63
Project: aws-lambda-decorators   Author: gridsmartercities   File: examples.py    MIT License 5 votes vote down vote up
def handle_exceptions_example():
    dynamodb = boto3.resource("dynamodb")
    table = dynamodb.Table("your_table_name")
    table.query(KeyConditionExpression=Key("user_id").eq("1234"))
    # ... 
Example 64
Project: aws-media-services-simple-vod-workflow   Author: aws-samples   File: mediainfo.py    Apache License 2.0 5 votes vote down vote up
def get_signed_url(expires_in, bucket, obj):
    """
    Generate a signed URL
    :param expires_in:  URL Expiration time in seconds
    :param bucket:
    :param obj:         S3 Key name
    :return:            Signed URL
    """
    s3_cli = boto3.client("s3")
    presigned_url = s3_cli.generate_presigned_url('get_object', Params={'Bucket': bucket, 'Key': obj},
                                                  ExpiresIn=expires_in)
    return presigned_url 
Example 65
Project: aws_lambda_for_python_example   Author: crazyfan5566   File: my_crawler_tw_temperature_process_data.py    MIT License 5 votes vote down vote up
def dynamodb_query(area_id):
    '''
    依條件查詢
    http://boto3.readthedocs.io/en/latest/guide/dynamodb.html#querying-and-scanning
    '''

    response = OBJ_TBL.query(
        KeyConditionExpression=Key('area_id').eq(area_id),
        ScanIndexForward=False,  # sort
        Limit=150
    )

    return response['Items'] 
Example 66
Project: nested-comments-serverless   Author: CodeRecipe-dev   File: dynamo_db_controller.py    MIT License 5 votes vote down vote up
def get_comments(self, recipe_id):
        response = self._recipes_metadata_table.query(
            KeyConditionExpression=Key('recipeId').eq(recipe_id),
            ScanIndexForward=False
        )
        items = response['Items']
        self._add_child_comments_to_parent_comments(items)
        return items 
Example 67
Project: nested-comments-serverless   Author: CodeRecipe-dev   File: dynamo_db_controller.py    MIT License 5 votes vote down vote up
def _get_thread_id(self,request_data):
        response = self._recipes_metadata_table.query(
            KeyConditionExpression=Key('recipeId').eq(request_data['recipeId'])
        )
        if response['Count'] == 0:
            threadId = 0
        else:
            threadId = response['Items'][-1]["threadId"] + 1
        return threadId 
Example 68
Project: streamalert   Author: airbnb   File: alert_table.py    Apache License 2.0 5 votes vote down vote up
def get_alert_records(self, rule_name, alert_proc_timeout_sec):
        """Find all alerts for the given rule which need to be dispatched to the alert processor.

        Args:
            rule_name (str): Select all alerts from this rule name
            alert_proc_timeout_sec (int): Alert processor timeout
                This is used to determine whether an alert could still be in progress

        Yields:
            dict: Each row in the Dynamo table which is not being worked on by the alert processor.
        """
        # Any alert which was recently dispatched to the alert processor may still be in progress,
        # so we'll skip over those for now.
        in_progress_threshold = datetime.utcnow() - timedelta(seconds=alert_proc_timeout_sec)

        kwargs = {
            # We need a consistent read here in order to pick up the most recent updates from the
            # alert processor. Otherwise, deleted/updated alerts may not yet have propagated.
            'ConsistentRead': True,

            # Include only those alerts which have not yet dispatched or were dispatched more than
            # ALERT_PROCESSOR_TIMEOUT seconds ago.
            'FilterExpression': (
                Attr('Dispatched').lt(in_progress_threshold.strftime(Alert.DATETIME_FORMAT))),

            'KeyConditionExpression': Key('RuleName').eq(rule_name)
        }
        for item in self._paginate(self._table.query, kwargs):
            yield item 
Example 69
Project: streamalert   Author: airbnb   File: alert_table.py    Apache License 2.0 5 votes vote down vote up
def update_sent_outputs(self, alert):
        """Update the table with the set of outputs which have sent successfully.

        Args:
            alert (Alert): Alert instance with sent outputs already updated.
        """
        self._table.update_item(
            Key=alert.dynamo_key,
            UpdateExpression='SET OutputsSent = :outputs_sent',
            ExpressionAttributeValues={':outputs_sent': alert.outputs_sent},
            ConditionExpression='attribute_exists(AlertID)'
        ) 
Example 70
Project: streamalert   Author: airbnb   File: alert_table.py    Apache License 2.0 5 votes vote down vote up
def delete_alerts(self, keys):
        """Remove an alert from the table.

        Args:
            keys (list): List of (rule_name, alert_id) str tuples
        """
        with self._table.batch_writer() as batch:
            for rule_name, alert_id in keys:
                batch.delete_item(Key={'RuleName': rule_name, 'AlertID': alert_id}) 
Example 71
Project: cloudtrail-tracker   Author: grycap   File: Querys.py    Apache License 2.0 5 votes vote down vote up
def read_table_item(table_name, pk_name, pk_value):
    """
    Return item read by primary key.
    """
    table = dynamodb_resource.Table(table_name)
    response = table.get_item(Key={pk_name: pk_value})



    return response 
Example 72
Project: cloudtrail-tracker   Author: grycap   File: Querys.py    Apache License 2.0 5 votes vote down vote up
def scan_table(table_name, filter_key=None, filter_value=None):
    """
    Perform a scan operation on table.
    Can specify filter_key (col name) and its value to be filtered.
    """
    table = dynamodb_resource.Table(table_name)

    if filter_key and filter_value:
        filtering_exp = Key(filter_key).eq(filter_value)
        response = table.scan(FilterExpression=filtering_exp)

        events = response['Items']
        while 'LastEvaluatedKey' in response:
            response = table.scan(ExclusiveStartKey=response['LastEvaluatedKey'],FilterExpression=filtering_exp)
            events.extend(response['Items'])

            # print(users)
    else:
        response = table.scan()

        events = response['Items']
        while 'LastEvaluatedKey' in response:
            response = table.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
            events.extend(response['Items'])

            # print(users)



    return events 
Example 73
Project: DataFS   Author: ClimateImpactLab   File: manager_dynamo.py    MIT License 5 votes vote down vote up
def _search(self, search_terms, begins_with=None):
        """
        Returns a list of Archive id's in the table on Dynamo

        """

        kwargs = dict(
            ProjectionExpression='#id',
            ExpressionAttributeNames={"#id": "_id"})

        if len(search_terms) > 0:
            kwargs['FilterExpression'] = reduce(
                lambda x, y: x & y,
                [Attr('tags').contains(arg) for arg in search_terms])

        if begins_with:
            if 'FilterExpression' in kwargs:
                kwargs['FilterExpression'] = kwargs[
                    'FilterExpression'] & Key('_id').begins_with(begins_with)

            else:
                kwargs['FilterExpression'] = Key(
                    '_id').begins_with(begins_with)

        while True:
            res = self._table.scan(**kwargs)
            for r in res['Items']:
                yield r['_id']
            if 'LastEvaluatedKey' in res:
                kwargs['ExclusiveStartKey'] = res['LastEvaluatedKey']
            else:
                break 
Example 74
Project: chaos   Author: th3sys   File: ig_executor.py    MIT License 4 votes vote down vote up
def UpdateStatus(self, order):
        update = 'UpdateStatus: '
        try:
            if order.Status == OrderStatus.Filled:
                trade = {
                  "FillTime": order.FillTime,
                  "Side": order.Side,
                  "FilledSize": decimal.Decimal(str(order.FillSize)),
                  "Price": decimal.Decimal(str(order.FillPrice)),
                  "Broker": {"Name": "IG", "RefType": "dealId", "Ref": order.BrokerReferenceId},
                  "StopDistance": order.StopDistance
                }
            if order.Status == OrderStatus.Failed:
                trade = {}

            response = self.__Orders.update_item(
                Key={
                    'OrderId': order.OrderId,
                    'TransactionTime': order.TransactionTime,
                },
                UpdateExpression="set #s = :s, Trade = :t",
                ConditionExpression="#s = :p",
                ExpressionAttributeNames={
                    '#s': 'Status'
                },
                ExpressionAttributeValues={
                    ':s': order.Status,
                    ':t': trade,
                    ':p': 'PENDING'
                },
                ReturnValues="UPDATED_NEW")
            update += '%s' % response['Attributes']

        except ClientError as e:
            self.__logger.error(e.response['Error']['Message'])
            update += e.response['Error']['Message']
        except Exception as e:
            self.__logger.error(e)
            update += e
        else:
            update += ". UpdateItem succeeded."
            self.__logger.info(response)

        self.__logger.info('Update: %s', update)
        return update 
Example 75
Project: chaos   Author: th3sys   File: ig_executor.py    MIT License 4 votes vote down vote up
def main(loop, logger, event):
    try:
        params = IGParams()
        params.Url = os.environ['IG_URL']
        params.Key = os.environ['X_IG_API_KEY']
        params.Identifier = os.environ['IDENTIFIER']
        params.Password = os.environ['PASSWORD']
        params.EAddress = os.environ['EMAIL_ADDRESS']
        params.EUser = os.environ['EMAIL_USER']
        params.EPassword = os.environ['EMAIL_PASSWORD']
        params.ESmtp = os.environ['EMAIL_SMTP']

        orders = []
        for record in event['Records']:
            if record['eventName'] == 'INSERT':
                orderId = record['dynamodb']['Keys']['OrderId']['S']
                logger.info('New Order received OrderId: %s', orderId)
                orders.append(record['dynamodb']['NewImage'])
            else:
                logger.info('Not INSERT event is ignored')
        if len(orders) == 0:
            logger.info('No Orders. Event is ignored')
            return

        async with Scheduler(params, logger, loop) as scheduler:

            valid, invalid = await scheduler.ValidateOrders(orders)
            if len(valid) == 0:
                scheduler.SendEmail('No Valid Security Definition has been found.')
                return
            logger.info('all validated orders %s' % [o.OrderId for o in valid])

            trades = await scheduler.GetPositions()

            passRisk = [order for order in valid if scheduler.BalanceCheck(order, trades)[1]]
            failedRisk = [order for order in valid if order not in passRisk]
            if len(passRisk) == 0:
                scheduler.SendEmail('No Security has been accepted by Risk Manager.')
                return
            logger.info('all passRisk orders %s' % [o.OrderId for o in passRisk])

            futures = [scheduler.SendOrder(o) for o in passRisk]
            done, _ = await asyncio.wait(futures, timeout=scheduler.Timeout)

            results = []
            for fut in done:
                name, payload = fut.result()
                results.append((name, payload))

            text = '<br>Orders where definition has not been found, not enabled for trading or not IG order %s\n' \
                   % invalid
            text += '<br>Orders where MaxPosition or RiskFactor in Securities table is exceeded %s\n' \
                    % [o.OrderId for o in failedRisk]
            text += '<br>The results of the trades sent to the IG REST API %s\n' % results
            scheduler.SendEmail(text)

    except Exception as e:
        logger.error(e) 
Example 76
Project: aws-media-services-vod-automation   Author: aws-samples   File: app.py    Apache License 2.0 4 votes vote down vote up
def status_progress(status):
    DYNAMO_TABLE_NAME = os.environ['JobTable']
    
    logger.info('GET progress for jobs with status {} from {}'.format(status, DYNAMO_TABLE_NAME))

    table = dynamodb_resource.Table(DYNAMO_TABLE_NAME)
    
    response = table.query(
        IndexName='status-createdAt-index', 
        KeyConditionExpression=Key('status').eq(status),
        ScanIndexForward=False
    )
    items =  response['Items']

    while True:
        print(len(response['Items']))
        if response.get('LastEvaluatedKey'):
            response = table.scan(
                ExclusiveStartKey=response['LastEvaluatedKey'],
                IndexName='status-createdAt-index', 
                KeyConditionExpression=Key('status').eq(status),
                ScanIndexForward=False
            )
            items += response['Items']
        else:
            break
    
    return items

# The view function above will return {"hello": "world"}
# whenever you make an HTTP GET request to '/'.
#
# Here are a few more examples:
#
# @app.route('/hello/{name}')
# def hello_name(name):
#    # '/hello/james' -> {"hello": "james"}
#    return {'hello': name}
#
# @app.route('/users', methods=['POST'])
# def create_user():
#     # This is the JSON body the user sent in their POST request.
#     user_as_json = app.current_request.json_body
#     # We'll echo the json body back to the user in a 'user' key.
#     return {'user': user_as_json}
#
# See the README documentation for more examples.
# 
Example 77
Project: aws-media-services-vod-automation   Author: aws-samples   File: mediainfo.py    Apache License 2.0 4 votes vote down vote up
def lambda_handler(event, context): 
    
    print(json.dumps(event))

    tsevent = int(datetime.datetime.strptime(event["time"], "%Y-%m-%dT%H:%M:%SZ").timestamp())
    
    try:
        # Get environment variables set on the CloudFormation stack
        MEDIAINFOTABLE = os.environ['MediainfoTable']
        MEDIAINFOTABLETTL = os.environ['MediainfoTableTTL']
        
        MEDIAINFO_RETENTION_PERIOD = (3600 * 24 * int(MEDIAINFOTABLETTL))

        # Loop through input videos in the event
        for input in event['detail']['inputDetails']:
            s3_path = input['uri']
            urlp = urlparse(s3_path)
            # Extract the Key and Bucket names for the inputs
            key = urlp[2]
            key = key.lstrip('/')
            bucket = urlp[1]
            
            signed_url = get_signed_url(SIGNED_URL_EXPIRATION, bucket, key)
            logger .info("Signed URL: {}".format(signed_url))
            
            print ("bucket and key "+bucket+" "+key)
            
            # Launch MediaInfo
            # Pass the signed URL of the uploaded asset to MediaInfo as an input
            # MediaInfo will extract the technical metadata from the asset
            # The extracted metadata will be outputted in XML format and
            # stored in the variable xml_output
            xml_output = subprocess.check_output(["./mediainfo", "--full", "--output=XML", signed_url])
            print(xml_output)
            
            json_output = xmltodict.parse(xml_output)
            
            input['mediainfo'] = json_output['Mediainfo']
            
            #print(json.dumps(json_output, indent=4, cls=DecimalEncoder))

        # add expirtation timestamp for dynamo and save the event in dynamo
        job_input_info = event['detail']
        job_input_info["timestamp"] = tsevent
        job_input_info["timestampTTL"] = tsevent + MEDIAINFO_RETENTION_PERIOD
        
        s = json.dumps(job_input_info, cls=DecimalEncoder)
        job_input_info = json.loads(s, parse_float=decimal.Decimal)
        table = DYNAMO_CLIENT.Table(MEDIAINFOTABLE)
        response = table.put_item(Item = job_input_info)
        print(json.dumps(response, cls=DecimalEncoder))
        
    except Exception as e:
        print('An error occured {}'.format(e))
        traceback.print_stack()
        raise
    else:
        return True 
Example 78
Project: open-house-crawler   Author: data-skeptic   File: OH-get-page-to-parse.py    GNU General Public License v3.0 4 votes vote down vote up
def lambda_handler(event, context):
    # TODO: can we randomize the result so simultaneous workers don't collide?
    # Maybe if it gets that frequent then we should be using SQS instead anyway.
    retry = 10
    response = table.query(
        IndexName='status-mask-index',
        KeyConditionExpression=Key('status-mask').eq(50), Limit=retry)
    #
    if response['Count'] == 0:
        s3key = 'empty.htm'
    else:
        items = response['Items']
        random.shuffle(items)
        ii = 0
        while ii < retry and ii < response['Count']:
            item = items[ii]
            url = item['uri']
            tld = tldextract_extract(url)
            if tld['subdomain'] != '' and tld['subdomain'] != 'www':
                tld = tld['subdomain'] + '.' + tld['domain'] + '.' + tld['suffix']
            else:
                tld = tld['domain'] + '.' + tld['suffix']
            i = url.find(tld)
            s3key = tld + url[i+len(tld):]
            print(s3key)
            try:
                obj = s3.Object(bucket, s3key)
                data = io.BytesIO()
                obj.download_fileobj(data)
                s = data.getvalue().decode("utf-8")
                resp = {"content": s, "url": url}
                return resp
            except:
                pass
            ii += 1
    s3key = 'empty.htm'
    obj = s3.Object(bucket, s3key)
    data = io.BytesIO()
    obj.download_fileobj(data)
    s = data.getvalue().decode("utf-8")
    resp = {"content": s, "url": "done"}
    return resp 
Example 79
Project: aws-media-services-simple-vod-workflow   Author: aws-samples   File: mediainfo.py    Apache License 2.0 4 votes vote down vote up
def lambda_handler(event, context): 
    
    print(json.dumps(event))

    tsevent = int(datetime.datetime.strptime(event["time"], "%Y-%m-%dT%H:%M:%SZ").timestamp())
    
    try:
        # Get environment variables set on the CloudFormation stack
        MEDIAINFOTABLE = os.environ['MediainfoTable']
        MEDIAINFOTABLETTL = os.environ['MediainfoTableTTL']
        
        MEDIAINFO_RETENTION_PERIOD = (3600 * 24 * int(MEDIAINFOTABLETTL))

        # Loop through input videos in the event
        for input in event['detail']['inputDetails']:
            s3_path = input['uri']
            urlp = urlparse(s3_path)
            # Extract the Key and Bucket names for the inputs
            key = urlp[2]
            key = key.lstrip('/')
            bucket = urlp[1]
            
            signed_url = get_signed_url(SIGNED_URL_EXPIRATION, bucket, key)
            logger .info("Signed URL: {}".format(signed_url))
            
            print ("bucket and key "+bucket+" "+key)
            
            # Launch MediaInfo
            # Pass the signed URL of the uploaded asset to MediaInfo as an input
            # MediaInfo will extract the technical metadata from the asset
            # The extracted metadata will be outputted in XML format and
            # stored in the variable xml_output
            xml_output = subprocess.check_output(["./mediainfo", "--full", "--output=XML", signed_url])
            print(xml_output)
            
            json_output = xmltodict.parse(xml_output)
            
            input['mediainfo'] = json_output['Mediainfo']
            
            #print(json.dumps(json_output, indent=4, cls=DecimalEncoder))

        # add expirtation timestamp for dynamo and save the event in dynamo
        job_input_info = event['detail']
        job_input_info["timestamp"] = tsevent
        job_input_info["timestampTTL"] = tsevent + MEDIAINFO_RETENTION_PERIOD
        
        s = json.dumps(job_input_info, cls=DecimalEncoder)
        job_input_info = json.loads(s, parse_float=decimal.Decimal)
        table = DYNAMO_CLIENT.Table(MEDIAINFOTABLE)
        response = table.put_item(Item = job_input_info)
        print(json.dumps(response, cls=DecimalEncoder))
        
    except Exception as e:
        print('An error occured {}'.format(e))
        traceback.print_stack()
        raise
    else:
        return True 
Example 80
Project: cloudtrail-tracker   Author: grycap   File: Write.py    Apache License 2.0 4 votes vote down vote up
def news(self, name_table, user, service, new_columns):
        """Get an user and add it if not exist yet"""
        index = 'userIdentity_userName'
        setValue_cols = 'cols'
        setValue_services  = 'services'
        setValue_listUsers = 'listUsers'
        arr = ['userIdentity_userName', 'all']
        # eventos = resource.Table(name_table)
        dynamodb = boto3.resource('dynamodb')
        table = dynamodb.Table(name_table)

        # filter expression
        fe = Key(arr[0]).eq(arr[1]);
        response = table.query(
            IndexName=self.index,
            KeyConditionExpression=fe,
        )
        cols = response['Items'][0][setValue_cols]
        services = response['Items'][0][setValue_services]
        users = response['Items'][0][setValue_listUsers]
        # users = ast.literal_eval(users)
        aux = False
        for column in new_columns:
            if cols.get(column, None) is None:
                aux = True
                cols[column] = '1'

        if users.get(user,None) is None:
            aux = True
            users[user] = '1'

        if services.get(service, None) is None:
            aux = True
            services[service] = '1'

        if not aux: return
        # update
        table.update_item(
            Key={
                'eventID': '1',
                'userIdentity_userName': 'all',
            },
            UpdateExpression=("SET {0} = :p, {1} = :r, {2} = :q").format(setValue_cols, setValue_services, setValue_listUsers),
            ExpressionAttributeValues={
                ':p': cols,
                ':r': services,
                ':q': users,
            },
            ReturnValues="UPDATED_NEW"
        )