Python boto3.dynamodb.conditions.Key() Examples

The following are 50 code examples for showing how to use boto3.dynamodb.conditions.Key(). They are extracted from open source Python projects. You can vote up the examples you like or vote down the exmaples you don't like. You can also save this page to your account.

Example 1
Project: kev   Author: capless   File: db.py    (license) View Source Project 7 votes vote down vote up
def parse_filters(self, filters, doc_class):
        index_name = None
        filter_expression_list = []
        query_params = {}
        for idx, filter in enumerate(filters):
            prop_name, prop_value = filter.split(':')[3:5]
            if idx == 0:
                prop = doc_class()._base_properties[prop_name]
                index_name = prop.kwargs.get(self.index_field_name, None) or \
                             self.default_index_name.format(prop_name)
                query_params['KeyConditionExpression'] = Key(prop_name).eq(prop_value)
            else:
                filter_expression_list.append(Attr(prop_name).eq(prop_value))
        if len(filter_expression_list) > 1:
            query_params['FilterExpression'] = And(*filter_expression_list)
        elif len(filter_expression_list) == 1:
            query_params['FilterExpression'] = filter_expression_list[0]
        if index_name != '_id':
            query_params['IndexName'] = index_name
        return query_params 
Example 2
Project: aws-ops-automator   Author: awslabs   File: task_tracking_table.py    (license) View Source Project 7 votes vote down vote up
def get_waiting_tasks(self, concurrency_key):
        """
        Returns list of waiting tasks with the specified concurrency key
        :param concurrency_key: concurrency key of the tasks
        :return: concurrency_key: list of waiting tasks
        """
        args = {
            "IndexName": "WaitForExecutionTasks",
            "Select": "ALL_ATTRIBUTES",
            "KeyConditionExpression": Key(TASK_TR_CONCURRENCY_ID).eq(concurrency_key),
            "FilterExpression": Attr(TASK_TR_STATUS).eq(STATUS_WAITING)
        }
        waiting_list = []
        while True:
            resp = self._action_table.query_with_retries(**args)
            waiting_list += resp.get("Items", [])

            last = resp.get("LastEvaluatedKey")

            if last is not None:
                args["ExclusiveStartKey"] = last
            else:
                break

        return waiting_list 
Example 3
Project: zappa-bittorrent-tracker   Author: Miserlou   File: track.py    (MIT License) View Source Project 6 votes vote down vote up
def remove_peer_from_info_hash_dynamo(
                            info_hash, 
                            peer_id, 
                        ):

    ensure_torrent_exists(info_hash)

    # Update the torrents list with the new information
    result = table.update_item(
        Key={
            'info_hash': info_hash,
        },
        UpdateExpression="REMOVE peers.#s",
        ExpressionAttributeNames={
            '#s': peer_id,
        },
        ReturnValues="UPDATED_NEW"
    )

    if result['ResponseMetadata']['HTTPStatusCode'] == 200 and 'Attributes' in result:
        return True
    return False 
Example 4
Project: zappa-bittorrent-tracker   Author: Miserlou   File: track.py    (MIT License) View Source Project 6 votes vote down vote up
def increment_completed_dynamo(info_hash):
    """
    Atomic increment completed for a torrent.
    """

    ensure_torrent_exists(info_hash)

    # Update the torrents list with the new information
    result = table.update_item(
        Key={
            'info_hash': info_hash,
        },
        UpdateExpression="SET completed = completed + :incr",
        ExpressionAttributeValues={
            ':incr': 1,
        },
        ReturnValues="UPDATED_NEW"
    )

    if result['ResponseMetadata']['HTTPStatusCode'] == 200 and 'Attributes' in result:
        return True
    return False 
Example 5
Project: donatemates   Author: donatemates   File: aws.py    (license) View Source Project 6 votes vote down vote up
def get_item(self, data):
        """Method to get an item

        Args:
            data (dict): A dictionary of attributes to put

        Returns:
            (dict)
        """
        try:
            response = self.table.get_item(Key=data,
                                           ConsistentRead=True)
        except ClientError as err:
            raise IOError("Error getting item: {}".format(err.message))

        if response['ResponseMetadata']['HTTPStatusCode'] != 200:
            raise IOError("Error getting item: {}".format(response['ResponseMetadata']))

        if "Item" in response:
            return response["Item"]
        else:
            return None 
Example 6
Project: donatemates   Author: donatemates   File: aws.py    (license) View Source Project 6 votes vote down vote up
def delete_item(self, data):
        """Method to get an item

        Args:
            data (dict): A dictionary of attributes to access an item (hash and sort keys)

        Returns:
            None
        """
        try:
            response = self.table.delete_item(Key=data)

        except ClientError as err:
            raise IOError("Error deleting item: {}".format(err.message))

        if response['ResponseMetadata']['HTTPStatusCode'] != 200:
            raise IOError("Error deleting item: {}".format(response['ResponseMetadata'])) 
Example 7
Project: donatemates   Author: donatemates   File: aws.py    (license) View Source Project 6 votes vote down vote up
def update_attribute(self, key_dict, attribute_name, attribute_value):
        """Method to update a single attribute in a record

        Args:
            key_dict (dict): A dictionary containing the keys/values to query on. Supports simple and compound keys
            attribute_name (str):
            attribute_value (str):

        Returns:
            None
        """
        response = self.table.update_item(Key=key_dict,
                                          UpdateExpression="SET {} = :updated".format(attribute_name),
                                          ExpressionAttributeValues={':updated': '{}'.format(attribute_value)})

        if response['ResponseMetadata']['HTTPStatusCode'] != 200:
            raise Exception("Error getting item: {}".format(response['ResponseMetadata']))

        # TODO: Check if any sort of validation on a update should done. DynamoDB seems lax here.
        #if "Attributes" in response:
        #    if len(response["Attributes"]) == 0:
        #        raise ValueError("Specified key does not exist. Update failed.")
        #else:
        #    raise ValueError("Specified key does not exist. Update failed.") 
Example 8
Project: donatemates   Author: donatemates   File: aws.py    (license) View Source Project 6 votes vote down vote up
def increment_attribute(self, key_dict, attribute_name, increment_value):
        """Method to increment a single attribute in a record

        Args:
            key_dict (dict): A dictionary containing the keys/values to query on. Supports simple and compound keys
            attribute_name (str): The attribute to increment
            increment_value (int): The amount to increment the attribute by

        Returns:
            None
        """
        response = self.table.update_item(Key=key_dict,
                                          UpdateExpression="SET {} = {} + :increment".format(attribute_name,
                                                                                             attribute_name),
                                          ExpressionAttributeValues={':increment': increment_value},
                                          ReturnValues="UPDATED_NEW")


        if response['ResponseMetadata']['HTTPStatusCode'] != 200:
            raise Exception("Error getting item: {}".format(response['ResponseMetadata'])) 
Example 9
Project: sqs-browser-events   Author: ReutersMedia   File: dynamo_sessions.py    (license) View Source Project 6 votes vote down vote up
def set_message_read(user_id, msg_id):
    try:
        r=get_history_table(new_session=True).update_item(
            Key={'userId':user_id,
                 'messageId':msg_id},
            UpdateExpression="set is_read = :a",
            ExpressionAttributeValues={':a': 1},
            ConditionExpression="is_read <> :a")
        LOGGER.info("Read-receipted user_id={0} message_id={1}".format(user_id,msg_id))
        return True
    except botocore.exceptions.ClientError as e:
        if e.response['Error']['Code'] == 'ConditionalCheckFailedException':
            LOGGER.info("Message already read user_id={0}, msg_id={1}".format(user_id,msg_id))
            return False
        else:
            LOGGER.exception("Eror updating read setting for user_id={0}, msg_id={1}".format(user_id,msg_id))
            return False 
Example 10
Project: cruddy   Author: Min-ops   File: __init__.py    (license) View Source Project 6 votes vote down vote up
def _analyze_table(self):
        # First check the Key Schema
        if len(self.table.key_schema) != 1:
            LOG.info('cruddy does not support RANGE keys')
        else:
            self._indexes[self.table.key_schema[0]['AttributeName']] = None
        # Now process any GSI's
        if self.table.global_secondary_indexes:
            for gsi in self.table.global_secondary_indexes:
                # find HASH of GSI, that's all we support for now
                # if the GSI has a RANGE, we ignore it for now
                if len(gsi['KeySchema']) == 1:
                    gsi_hash = gsi['KeySchema'][0]['AttributeName']
                    self._indexes[gsi_hash] = gsi['IndexName']

    # Because the Boto3 DynamoDB client turns all numeric types into Decimals
    # (which is actually the right thing to do) we need to convert those
    # Decimal values back into integers or floats before serializing to JSON. 
Example 11
Project: cruddy   Author: Min-ops   File: __init__.py    (license) View Source Project 6 votes vote down vote up
def increment_counter(self, id, counter_name, increment=1,
                          id_name='id', **kwargs):
        """
        Atomically increments a counter attribute in the item identified by
        ``id``.  You must specify the name of the attribute as ``counter_name``
        and, optionally, the ``increment`` which defaults to ``1``.
        """
        response = self._new_response()
        if self._check_supported_op('increment_counter', response):
            params = {
                'Key': {id_name: id},
                'UpdateExpression': 'set #ctr = #ctr + :val',
                'ExpressionAttributeNames': {"#ctr": counter_name},
                'ExpressionAttributeValues': {
                    ':val': decimal.Decimal(increment)},
                'ReturnValues': 'UPDATED_NEW'
            }
            self._call_ddb_method(self.table.update_item, params, response)
            if response.status == 'success':
                if 'Attributes' in response.raw_response:
                    self._replace_decimals(response.raw_response)
                    attr = response.raw_response['Attributes'][counter_name]
                    response.data = attr
        response.prepare()
        return response 
Example 12
Project: ebs_snapper   Author: rackerlabs   File: dynamo.py    (license) View Source Project 6 votes vote down vote up
def list_ids(context, installed_region, aws_account_id=None):
    """Retrieve configuration from DynamoDB and return array of dictionary objects"""
    found_configurations = {}
    if aws_account_id is None:
        aws_account_id = utils.get_owner_id(context)[0]

    dynamodb = boto3.resource('dynamodb', region_name=installed_region)
    table = dynamodb.Table('ebs_snapshot_configuration')

    results = table.query(
        KeyConditionExpression=Key('aws_account_id').eq(aws_account_id)
    )

    for item in results.get('Items', []):
        str_item = item.get('configuration', None)
        found_configurations[str_item] = item['id']

    return found_configurations.values() 
Example 13
Project: ebs_snapper   Author: rackerlabs   File: dynamo.py    (license) View Source Project 6 votes vote down vote up
def list_configurations(context, installed_region, aws_account_id=None):
    """Retrieve configuration from DynamoDB and return array of dictionary objects"""
    found_configurations = {}
    if aws_account_id is None:
        aws_account_id = utils.get_owner_id(context)[0]

    dynamodb = boto3.resource('dynamodb', region_name=installed_region)
    table = dynamodb.Table('ebs_snapshot_configuration')

    results = table.query(
        KeyConditionExpression=Key('aws_account_id').eq(aws_account_id)
    )

    for item in results.get('Items', []):
        str_item = item.get('configuration', None)
        try:
            json_item = json.loads(str_item)
            found_configurations[str_item] = json_item
        except Exception as e:
            raise EbsSnapperError('error loading configuration', e)

    return found_configurations.values() 
Example 14
Project: ebs_snapper   Author: rackerlabs   File: dynamo.py    (license) View Source Project 6 votes vote down vote up
def get_configuration(context, installed_region, object_id, aws_account_id=None):
    """Retrieve configuration from DynamoDB and return single object"""
    if aws_account_id is None:
        aws_account_id = utils.get_owner_id(context)[0]

    dynamodb = boto3.resource('dynamodb', region_name=installed_region)
    table = dynamodb.Table('ebs_snapshot_configuration')

    expr = Key('aws_account_id').eq(aws_account_id) & Key('id').eq(object_id)
    results = table.query(KeyConditionExpression=expr)

    for item in results.get('Items', []):
        str_item = item.get('configuration', None)
        try:
            json_item = json.loads(str_item)
            return json_item
        except Exception as e:
            raise EbsSnapperError('error loading configuration', e)

    return None 
Example 15
Project: boss   Author: jhuapl-boss   File: metadb.py    (license) View Source Project 6 votes vote down vote up
def write_meta(self, lookup_key, key, value):
        """
        Write the  meta data to dyanmodb
        Args:
            lookup_key: Key for the object requested
            key: Meta data key
            value: Metadata value

        Returns:

        """

        response = self.table.put_item(
            Item={
                'lookup_key': lookup_key,
                'key': key,
                'metavalue': value,
            }
        )
        return response 
Example 16
Project: boss   Author: jhuapl-boss   File: metadb.py    (license) View Source Project 6 votes vote down vote up
def get_meta(self, lookup_key, key):
        """
        Retrieve the meta data for a given key
        Args:
            lookup_key: Key for the object requested
            key: Metadata key

        Returns:

        """

        response = self.table.get_item(
            Key={
                'lookup_key': lookup_key,
                'key': key,
            }
        )
        if 'Item' in response:
            return response['Item']
        else:
            return None 
Example 17
Project: boss   Author: jhuapl-boss   File: metadb.py    (license) View Source Project 6 votes vote down vote up
def update_meta(self, lookup_key, key, new_value):
        """
        Update the Value for the given key
        Args:
            lookup_key: Key for the object requested
            key: Metadata key
            new_value: New meta data value

        Returns:

        """

        response = self.table.update_item(
            Key={
                'lookup_key': lookup_key,
                'key': key,
            },
            UpdateExpression='SET metavalue = :val1',
            ExpressionAttributeValues={
                ':val1': new_value
            },
            ReturnValues='UPDATED_NEW'
        )
        return response 
Example 18
Project: boss   Author: jhuapl-boss   File: metadb.py    (license) View Source Project 6 votes vote down vote up
def get_meta_list(self, lookup_key):
        """
        Retrieve all the meta data for a given object using the lookupley
        Args:
            lookup_key: Key for the object requested
        Returns:

        """
        response = self.table.query(
            KeyConditionExpression=Key('lookup_key').eq(lookup_key)
        )

        if 'Items' in response:
            return response['Items']
        else:
            return None 
Example 19
Project: Project-Violet-Sloth   Author: Violet-Sloth   File: chuckie.py    (license) View Source Project 6 votes vote down vote up
def lambda_handler(event, context):
    wotd = table.query(
        KeyConditionExpression=Key('language').eq(event['request']['intent']['slots']["Language"]["value"].lower()) & Key('id').gt(0),
        FilterExpression=Attr('isActive').eq(True)
    )
    print(wotd['Items'])
    item = wotd["Items"]
    print(item)
    parsed = '<speak>The ' + event["request"]["intent"]["slots"]["Language"]["value"] + ' word of the day is <audio src="' + item[0]["word_sound"] + \
    '"/> which means ' + item[0]["word_translation"] + \
    '. Here is the word used in a sentence. <audio src="' + item[0]["sentence_sound"] + \
    '"/> which means ' + item[0]["sentence_translation"] + '</speak>'
    response = {
        'version': '1.0',
        'response': {
            'outputSpeech': {
                'type': 'SSML',
                'ssml' : parsed
            }
        }
    }
    return response 
Example 20
Project: dynamodb-py   Author: gusibi   File: query_items.py    (license) View Source Project 6 votes vote down vote up
def query_with_limit_and_filter_by_boto3():
    table = dynamodb.Table('Movies')
    print("Movies from 1992 - titles A-L, with genres and lead actor")
    try:
        response = table.query(
            ProjectionExpression="#yr, title, info.genres, info.actors[0]",
            ExpressionAttributeNames={"#yr": "year"}, # Expression Attribute Names for Projection Expression only.
            KeyConditionExpression=Key('year').eq(1992) & Key('title').between('A', 'L'),
            FilterExpression=Attr('rating').lt(decimal.Decimal(str('7.0'))),
            Limit=10,
        )
    except ClientError as e:
        print(e.response['Error']['Message'])
    else:
        items = response['Items']
        for i in items:
            print(i['year'], ":", i['title']) 
Example 21
Project: mentii   Author: mentii   File: user_ctrl.py    (license) View Source Project 6 votes vote down vote up
def updatePasswordForEmailAndResetId(email, password, resetPasswordId, dbInstance):
  res = None
  user = getUserByEmail(email, dbInstance)
  if user is not None:
    storedResetPasswordId = user.get('resetPasswordId', None)
    if storedResetPasswordId == resetPasswordId:
      table = dbUtils.getTable('users', dbInstance)
      if table is not None:
        hashedPassword = hashPassword(password)
        jsonData = {
          'Key': {'email': email},
          'UpdateExpression': 'SET password = :a REMOVE resetPasswordId',
          'ExpressionAttributeValues': { ':a': hashedPassword },
          'ReturnValues' : 'UPDATED_NEW'
        }
        res = dbUtils.updateItem(jsonData, table)
  return res 
Example 22
Project: mentii   Author: mentii   File: user_ctrl.py    (license) View Source Project 6 votes vote down vote up
def joinClass(jsonData, dynamoDBInstance, email=None, userRole=None):
  response = ControllerResponse()
  #g will be not be available during testing
  #and email will need to be passed to the function
  if g: # pragma: no cover
    email = g.authenticatedUser['email']
    userRole = g.authenticatedUser['userRole']

  if 'code' not in jsonData.keys() or not jsonData['code']:
    response.addError('Key Missing Error', 'class code missing from data')
  elif userRole == 'teacher' or userRole == 'admin':
    if class_ctrl.isCodeInTaughtList(jsonData, dynamoDBInstance, email):
      response.addError('Role Error', 'Teachers cannot join their taught class as a student')
    else:
      classCode = jsonData['code']
      addDataToClassAndUser(classCode, email, response, dynamoDBInstance)
  else:
    classCode = jsonData['code']
    addDataToClassAndUser(classCode, email, response, dynamoDBInstance)
  return response 
Example 23
Project: mentii   Author: mentii   File: user_ctrl.py    (license) View Source Project 6 votes vote down vote up
def addClassCodeToStudent(email, classCode, dynamoDBInstance):
  userTable = dbUtils.getTable('users', dynamoDBInstance)
  if userTable:
    codeSet = set([classCode])
    addClassToUser = {
      'Key': {'email': email},
      'UpdateExpression': 'ADD classCodes :i',
      'ExpressionAttributeValues': { ':i': codeSet },
      'ReturnValues' : 'UPDATED_NEW'
    }
    res = dbUtils.updateItem(addClassToUser, userTable)
    if (  res and
          'Attributes' in res and
          'classCodes' in res['Attributes'] and
          classCode in res['Attributes']['classCodes']
    ):
      return res['Attributes']['classCodes']
  return None 
Example 24
Project: mentii   Author: mentii   File: user_ctrl.py    (license) View Source Project 6 votes vote down vote up
def addStudentToClass(classCode, email, dynamoDBInstance):
  classTable = dbUtils.getTable('classes', dynamoDBInstance)
  if classTable:
    emailSet = set([email])
    addUserToClass = {
      'Key': {'code': classCode},
      'UpdateExpression': 'ADD students :i',
      'ExpressionAttributeValues': { ':i': emailSet },
      'ReturnValues' : 'ALL_NEW'
    }
    res = dbUtils.updateItem(addUserToClass, classTable)
    if (  res and
          'Attributes' in res and
          'students' in res['Attributes'] and
          email in res['Attributes']['students'] and
          'title' in res['Attributes']
    ):
      return res['Attributes']
  return None 
Example 25
Project: mentii   Author: mentii   File: class_ctrl.py    (license) View Source Project 6 votes vote down vote up
def getActiveClassList(dynamoDBInstance, email=None):
  response = ControllerResponse()
  usersTable = dbUtils.getTable('users', dynamoDBInstance)
  classTable = dbUtils.getTable('classes', dynamoDBInstance)

  if usersTable is None or classTable is None:
    response.addError(  'Get Active Class List Failed',
                        'Unable to access users and/or classes')
  else :
    if email is None: # pragma: no cover
      email = g.authenticatedUser['email']
    classes = []
    classCodes = getClassCodesFromUser(dynamoDBInstance, email)

    for code in classCodes:
      request = {'Key': {'code': code}}
      res = dbUtils.getItem(request, classTable)
      if res is not None and 'Item' in res:
        classes.append(res['Item'])
    response.addToPayload('classes', classes)
  return response 
Example 26
Project: mentii   Author: mentii   File: class_ctrl.py    (license) View Source Project 6 votes vote down vote up
def getClassCodesFromUser(dynamoDBInstance, email=None):
  classCodes = set()
  if email is None: # pragma: no cover
    email = g.authenticatedUser['email']
  usersTable = dbUtils.getTable('users', dynamoDBInstance)
  if usersTable is None:
    MentiiLogging.getLogger().error('Unable to get users table in getClassCodesFromUser')
  else:
    #An active class list is the list of class codes that
    # a user has in the user table.
    request = {"Key" : {"email": email}, "ProjectionExpression": "classCodes"}
    res = dbUtils.getItem(request, usersTable)
    #Get the class codes for the user.
    if res is None or 'Item' not in res or 'classCodes' not in res['Item']:
      MentiiLogging.getLogger().error('Unable to get user data in getClassCodesFromUser')
    else:
      classCodes = res['Item']['classCodes']
  return classCodes 
Example 27
Project: mentii   Author: mentii   File: class_ctrl.py    (license) View Source Project 6 votes vote down vote up
def getTaughtClassCodesFromUser(dynamoDBInstance, email=None):
  classCodes = None
  if email is None: # pragma: no cover
    email = g.authenticatedUser['email']
  usersTable = dbUtils.getTable('users', dynamoDBInstance)
  if usersTable is None:
    MentiiLogging.getLogger().error('Unable to get users table in getTaughtClassCodesFromUser')
  else:
    #An active class list is the list of class codes that
    # a user has in the user table.
    request = {'Key' : {'email': email}, 'ProjectionExpression': 'teaching'}
    res = dbUtils.getItem(request, usersTable)
    #Get the class codes for the user.
    if res is not None and 'Item' in res:
      classCodes = res['Item'].get('teaching', [])
  return classCodes 
Example 28
Project: mentii   Author: mentii   File: class_ctrl.py    (license) View Source Project 6 votes vote down vote up
def buildUpdateJsonData(keyName, keyValue, attributeName, attributeValue):
  jsonData = {}
  if len(attributeValue) == 0:
    #remove attribute
    jsonData = {
      'Key': {keyName : keyValue},
      'UpdateExpression': 'REMOVE '+ attributeName,
      'ReturnValues' : 'UPDATED_NEW'
    }
  else:
    #update attribute
    jsonData = {
      'Key': {keyName : keyValue},
      'UpdateExpression': 'SET ' + attributeName + ' = :v',
      'ExpressionAttributeValues': { ':v': attributeValue },
      'ReturnValues' : 'UPDATED_NEW'
    }
  return jsonData 
Example 29
Project: mentii   Author: mentii   File: db_utils.py    (license) View Source Project 6 votes vote down vote up
def getItem(jsonData, table):
  if (type(jsonData) == str):
    data = json.loads(jsonData)
  else:
    data = jsonData

  projection_expression = data.get("ProjectionExpression")
  key = data.get("Key")

  if key is None:
    message = "Unable to get item. Missing Key"
    logger.error(message)
    return None

  if projection_expression is not None:
    response = table.get_item(Key=key,ProjectionExpression=projection_expression)
  else:
    response = table.get_item(Key=key)
  return response 
Example 30
Project: awslimits   Author: Yipit   File: support.py    (license) View Source Project 6 votes vote down vote up
def update_ticket(form):
    table = get_tickets_table()
    limit_type = form.limit_type.data
    table.update_item(
        Key={
            "display_id": form.display_id.data,
        },
        AttributeUpdates={
            'limit_type': {
                'Value': limit_type,
                'Action': 'PUT',
            },
            'limit_value': {
                'Value': form.limit_value.data,
                'Action': 'PUT',
            },
    })
    update_limit_value(limit_type) 
Example 31
Project: AWS_EC2_Scheduler   Author: tangerinedream   File: Orchestrator.py    (license) View Source Project 6 votes vote down vote up
def lookupWorkloadSpecification(self, partitionTargetValue):
		try:
			dynamodbItem=self.dynDBC.get_item(
				TableName=self.workloadSpecificationTableName,
				Key={
					self.workloadSpecificationPartitionKey : { "S" : partitionTargetValue }
				},
				ConsistentRead=False,
				ReturnConsumedCapacity="TOTAL",
			)
		except ClientError as e:
			self.logger.error('lookupWorkloadSpecification()' + e.response['Error']['Message'])
		else:
			# Get the dynamoDB Item from the result
			resultItem=dynamodbItem['Item']

		
			for attributeName in resultItem:
				# Validate the attributes entered into DynamoDB are valid.  If not, spit out individual warning messages
				if( attributeName in self.workloadSpecificationValidAttributeList ):
					attributeValue=resultItem[attributeName].values()[0]
					self.logger.info('Workload Attribute [%s maps to %s]' % (attributeName, attributeValue))
					self.workloadSpecificationDict[attributeName]=attributeValue
				else:
					self.logger.warning('Invalid dynamoDB attribute specified->'+str(attributeName)+'<- will be ignored') 
Example 32
Project: zappa-bittorrent-tracker   Author: Miserlou   File: track.py    (MIT License) View Source Project 5 votes vote down vote up
def purge_expired_peers():
    """
    Removes peers who haven't announced in the last internval.

    Should be set as a recurring event source in your Zappa config.
    """

    if DATASTORE == "DynamoDB":
        # This is a costly operation, but I think it has to be done.
        # Optimizations (pagination? queries? batching?) welcomed.
        all_torrents = table.scan()
        for torrent in all_torrents['Items']:
            for peer_id in torrent['peers']:
                peer_last_announce = int(torrent['peers'][peer_id][0]['last_announce'])
                window = datetime.now() - timedelta(seconds=ANNOUNCE_INTERVAL)
                window_unix = int(time.mktime(window.timetuple()))

                if peer_last_announce < window_unix:
                    remove_peer_from_info_hash(torrent['info_hash'], peer_id)
    else:
        # There must be a better way to do this.
        # Also, it should probably be done as a recurring function and cache,
        # not dynamically every time.
        for key in s3_client.list_objects(Bucket=BUCKET_NAME)['Contents']:
            if 'peers.json' in key['Key']:
                remote_object = s3.Object(BUCKET_NAME, key['Key']).get()
                content = remote_object['Body'].read().decode('utf-8')
                torrent = json.loads(content)
                for peer_id in torrent['peers']:
                    peer_last_announce = int(torrent['peers'][peer_id]['last_announce'])
                    window = datetime.now() - timedelta(seconds=ANNOUNCE_INTERVAL)
                    window_unix = int(time.mktime(window.timetuple()))

                    if peer_last_announce < window_unix:
                        remove_peer_from_info_hash(torrent['info_hash'], peer_id)

    return

##
# Database
## 
Example 33
Project: zappa-bittorrent-tracker   Author: Miserlou   File: track.py    (MIT License) View Source Project 5 votes vote down vote up
def get_peers_for_info_hash_dynamodb(
                info_hash, 
                limit=50
            ):
    """
    Get current peers
    """

    response = table.query(
        KeyConditionExpression=Key('info_hash').eq(info_hash)
    )
    if response['Count'] == 0:
        return []
    else:
        return response['Items'][0]['peers'] 
Example 34
Project: zappa-bittorrent-tracker   Author: Miserlou   File: track.py    (MIT License) View Source Project 5 votes vote down vote up
def get_all_items():
    """
    Get all items
    """

    if DATASTORE == "DynamoDB":
        response = table.scan()
        if response['Count'] == 0:
            return []
        else:
            return response['Items']
    else:

        # We want info_hash, peers, and completed.
        items = []
        
        # There must be a better way to do this.
        # Also, it should probably be done as a recurring function and cache,
        # not dynamically every time.
        for key in s3_client.list_objects(Bucket=BUCKET_NAME)['Contents']:
            if 'peers.json' in key['Key']:
                remote_object = s3.Object(BUCKET_NAME, key['Key']).get()
                content = remote_object['Body'].read().decode('utf-8')
                torrent_info = json.loads(content)
                item = {
                    'info_hash': torrent_info['info_hash'],
                    'peers': torrent_info['peers'],
                    'completed': torrent_info['completed']
                }
                items.append(item)

        return items

###
# Utility
###

# Helper class to convert a DynamoDB item to JSON. 
Example 35
Project: aws-tailor   Author: alanwill   File: handler.py    (GNU General Public License v3.0) View Source Project 5 votes vote down vote up
def freeprefix(nipap_deamon_ip, account_cb_alias, account_iam_alias, vpc_network, vpc_prefix):
    # Lookup nipap daemon password cipher
    nipapCfn = dynamodb.Table(os.environ['TAILOR_TABLENAME_NIPAPCFN'])
    getNipapCfn = nipapCfn.get_item(
        Key={
            'nipapAlias': account_cb_alias
        }
    )

    # Decrypt nipap daemon password
    nipapDaemonPasswordCipherBlob = getNipapCfn['Item']['nipapDaemonPasswordCipherBlob']
    nipapDeamonPassword = bytes(kms.decrypt(CiphertextBlob=b64decode(nipapDaemonPasswordCipherBlob))['Plaintext'])

    # Look up free CIDR block
    pynipap.xmlrpc_uri = "http://tailor:" + nipapDeamonPassword.rstrip() + "@" + nipap_deamon_ip + ":1337"

    a = pynipap.AuthOptions({
        'authoritative_source': 'tailor_nipap_client'
    })

    # Allocate first available
    new_prefix = Prefix()
    new_prefix.description = account_iam_alias
    new_prefix.type = "assignment"

    # Save will communicate with the backend and ask for the next available desired prefix size
    new_prefix.save({'from-prefix': [vpc_network], 'prefix_length': vpc_prefix})

    # Read the assigned prefix from the new_prefix object
    print("VPC Cidr is: ", new_prefix.prefix)
    return new_prefix.prefix 
Example 36
Project: aws-tailor   Author: alanwill   File: handler.py    (GNU General Public License v3.0) View Source Project 5 votes vote down vote up
def add_dhcp_optionset(la_credentials, la_vpc_id, dns_server_1, dns_server_2, region):

    # Initiate linked account ec2 client
    laEc2 = boto3.client(
        'ec2',
        region_name=region,
        aws_access_key_id=la_credentials[0],
        aws_secret_access_key=la_credentials[1],
        aws_session_token=la_credentials[2],
    )

    if region == 'us-east-1':
        domainName = 'ec2.internal'
    else:
        domainName = region + '.compute.internal'
    dhcpOptionsset = laEc2.create_dhcp_options(
        DhcpConfigurations=[
            {
                'Key': 'domain-name-servers',
                'Values': [dns_server_1, dns_server_2, 'AmazonProvidedDNS']
            },
            {
                'Key': 'domain-name',
                'Values': [domainName]
            },
        ],
    )

    laEc2.associate_dhcp_options(
        DhcpOptionsId=dhcpOptionsset['DhcpOptions']['DhcpOptionsId'],
        VpcId=la_vpc_id,
    )

    return 
Example 37
Project: DataFS   Author: ClimateImpactLab   File: manager_dynamo.py    (MIT License) View Source Project 5 votes vote down vote up
def _search(self, search_terms, begins_with=None):
        """
        Returns a list of Archive id's in the table on Dynamo

        """

        kwargs = dict(
            ProjectionExpression='#id',
            ExpressionAttributeNames={"#id": "_id"})

        if len(search_terms) > 0:
            kwargs['FilterExpression'] = reduce(
                lambda x, y: x & y,
                [Attr('tags').contains(arg) for arg in search_terms])

        if begins_with:
            if 'FilterExpression' in kwargs:
                kwargs['FilterExpression'] = kwargs[
                    'FilterExpression'] & Key('_id').begins_with(begins_with)

            else:
                kwargs['FilterExpression'] = Key(
                    '_id').begins_with(begins_with)

        while True:
            res = self._table.scan(**kwargs)
            for r in res['Items']:
                yield r['_id']
            if 'LastEvaluatedKey' in res:
                kwargs['ExclusiveStartKey'] = res['LastEvaluatedKey']
            else:
                break 
Example 38
Project: DataFS   Author: ClimateImpactLab   File: manager_dynamo.py    (MIT License) View Source Project 5 votes vote down vote up
def _update(self, archive_name, version_metadata):
        '''
        Updates the version specific metadata attribute in DynamoDB
        In DynamoDB this is simply a list append on this attribute value

        Parameters
        ----------
        archive_name: str
            unique '_id' primary key

        version_metadata: dict
            dictionary of version metadata values

        Returns
        -------
        dict
            list of dictionaries of version_history
        '''

        command = "SET version_history = list_append(version_history, :v)"

        self._table.update_item(
            Key={'_id': archive_name},
            UpdateExpression=command,
            ExpressionAttributeValues={':v': [version_metadata]},
            ReturnValues='ALL_NEW') 
Example 39
Project: DataFS   Author: ClimateImpactLab   File: manager_dynamo.py    (MIT License) View Source Project 5 votes vote down vote up
def _update_spec_config(self, document_name, spec):
        '''
        Dynamo implementation of project specific metadata spec

        '''
        # add the updated archive_metadata object to Dynamo
        self._spec_table.update_item(
            Key={'_id': '{}'.format(document_name)},
            UpdateExpression="SET config = :v",
            ExpressionAttributeValues={':v': spec},
            ReturnValues='ALL_NEW') 
Example 40
Project: DataFS   Author: ClimateImpactLab   File: manager_dynamo.py    (MIT License) View Source Project 5 votes vote down vote up
def _update_metadata(self, archive_name, archive_metadata):
        """
        Appends the updated_metada dict to the Metadata Attribute list

        Parameters
        ----------
        archive_name: str

            ID of archive to update

        updated_metadata: dict

            dictionary of metadata keys and values to update. If the value
            for a particular key is `None`, the key is removed.

        """

        archive_metadata_current = self._get_archive_metadata(archive_name)
        archive_metadata_current.update(archive_metadata)
        for k, v in archive_metadata_current.items():
            if v is None:
                del archive_metadata_current[k]

        # add the updated archive_metadata object to Dynamo
        self._table.update_item(
            Key={'_id': archive_name},
            UpdateExpression="SET archive_metadata = :v",
            ExpressionAttributeValues={':v': archive_metadata_current},
            ReturnValues='ALL_NEW') 
Example 41
Project: DataFS   Author: ClimateImpactLab   File: manager_dynamo.py    (MIT License) View Source Project 5 votes vote down vote up
def _get_archive_listing(self, archive_name):
        '''
        Return full document for ``{_id:'archive_name'}``

        .. note::

            DynamoDB specific results - do not expose to user
        '''
        return self._table.get_item(Key={'_id': archive_name})['Item'] 
Example 42
Project: DataFS   Author: ClimateImpactLab   File: manager_dynamo.py    (MIT License) View Source Project 5 votes vote down vote up
def _set_tags(self, archive_name, updated_tag_list):

        self._table.update_item(
                Key={'_id': archive_name},
                UpdateExpression="SET tags = :t",
                ExpressionAttributeValues={':t': updated_tag_list},
                ReturnValues='ALL_NEW') 
Example 43
Project: mafia-serverless   Author: akranga   File: main.py    (Apache License 2.0) View Source Project 5 votes vote down vote up
def clear_all():
  for i in find_all():
    table.delete_item(Key={'Name': i['Name']}) 
Example 44
Project: donatemates   Author: donatemates   File: aws.py    (license) View Source Project 5 votes vote down vote up
def query_hash(self, hash_name, hash_value, index=None, forward=True, limit=None, projection=None):
        """Method to query an index

        Args:
            data (dict): A dictionary of attributes to put

        Returns:
            (dict)
        """
        params = {"ScanIndexForward": forward,
                  "KeyConditionExpression": Key(hash_name).eq(hash_value)}
        if index:
            params["IndexName"] = index
        else:
            # If primary index, consistent read
            params["ConsistentRead"] = True

        if limit:
            params["Limit"] = limit

        if projection:
            params["ProjectionExpression"] = projection

        response = self.table.query(**params)

        if response['ResponseMetadata']['HTTPStatusCode'] != 200:
            raise Exception("Error getting item: {}".format(response['ResponseMetadata']))

        return response["Items"] 
Example 45
Project: donatemates   Author: donatemates   File: aws.py    (license) View Source Project 5 votes vote down vote up
def query_24hr(self, hash_key, hash_value, sort_key, date_str):
        """Method to query for a date range for the day provided in the date_str

        Date_str must be in ISO-8601

        This assumes you are "centering" the 24hr block from midnight-midnight EST

        Args:
            hash_key (str): Hash key name
            sort_key (str): Sort key name
            date_str (str): The date string containing the day to query in UTC time

        Returns:
            list
        """
        # Convert ISO time to be EST
        date_in = arrow.get(date_str)
        date_in_est = date_in.to('EST')

        # Compute start date str
        start_date = date_in_est.replace(hour=0, minute=0)

        # Compute end date str
        date_range = start_date.span('day')

        response = self.table.update_item(Key={hash_key: hash_value},
                                          KeyConditionExpression="{} >= :morning AND {} <= :midnight".format(sort_key,
                                                                                                             sort_key),
                                          ExpressionAttributeValues={':morning': date_range[0].isoformat(),
                                                                     ':midnight': date_range[1].isoformat()},
                                          ReturnValues="UPDATED_NEW")

        if response['ResponseMetadata']['HTTPStatusCode'] != 200:
            raise Exception("Error getting item: {}".format(response['ResponseMetadata']))

        if "Items" in response:
            return response["Items"]
        else:
            return [] 
Example 46
Project: donatemates   Author: donatemates   File: aws.py    (license) View Source Project 5 votes vote down vote up
def query_most_recent(self, hash_key, hash_value, sort_key, date_str, limit=1):
        """Method to query for the record most recently in the past based on the date_str

        Date_str must be in ISO-8601

        Args:
            hash_key (str): Hash key name
            sort_key (str): Sort key name
            date_str (str): The date string containing the day to query in UTC time

        Returns:
            dict
        """
        response = self.table.query(KeyConditionExpression=Key(hash_key).eq(hash_value) & Key(sort_key).lte(date_str),
                                    Limit=limit,
                                    ScanIndexForward=False,
                                    Select="ALL_ATTRIBUTES")

        if response['ResponseMetadata']['HTTPStatusCode'] != 200:
            raise Exception("Error getting item: {}".format(response['ResponseMetadata']))

        if "Items" in response:
            if response["Items"]:
                return response["Items"]
            else:
                return []
        else:
            return [] 
Example 47
Project: donatemates   Author: donatemates   File: aws.py    (license) View Source Project 5 votes vote down vote up
def query_biggest(self, hash_key, hash_value, num_items, index=None, forward=False):
        """Method to query for the largest N records

        Args:
            hash_key (str): Hash key name
            hash_value (str): Hash key value
            num_items (int): The number of items to return
            index (str): Name of index if not primary
            forward (bool): flag indicating sort direction

        Returns:
            dict
        """
        params = {"ScanIndexForward": forward,
                  "KeyConditionExpression": Key(hash_key).eq(hash_value),
                  "Select": "ALL_ATTRIBUTES",
                  "Limit": num_items}

        if index:
            params["IndexName"] = index
            params["Select"] = "ALL_PROJECTED_ATTRIBUTES"
        else:
            # If primary index, consistent read
            params["ConsistentRead"] = True

        response = self.table.query(**params)

        if response['ResponseMetadata']['HTTPStatusCode'] != 200:
            raise Exception("Error getting item: {}".format(response['ResponseMetadata']))

        if "Items" in response:
            if response["Items"]:
                return response["Items"]
            else:
                return []
        else:
            return [] 
Example 48
Project: custom-lookup-lambda   Author: awslabs   File: function.py    (license) View Source Project 5 votes vote down vote up
def get_data(event):
    dynamodb = boto3.resource('dynamodb')
    table = dynamodb.Table(tablename)
    response = table.query(
        KeyConditionExpression=Key(hashkey).eq(event[hashkey]) \
                               & Key(rangekey).eq(event[rangekey])
    )
    for item in response['Items']:
        objkeypair = ast.literal_eval(item['mappings'])
        if 'lookup' in event:
            return objkeypair[event['lookup']]
        else:
            return objkeypair 
Example 49
Project: kev   Author: capless   File: db.py    (license) View Source Project 5 votes vote down vote up
def delete(self, doc_obj):
        self._indexer.delete_item(Key={'_id': doc_obj._data['_id']}) 
Example 50
Project: kev   Author: capless   File: db.py    (license) View Source Project 5 votes vote down vote up
def get(self, doc_obj, doc_id):
        response = self._indexer.get_item(Key={'_id': doc_obj.get_doc_id(doc_id)})
        doc = response.get('Item', None)
        if not doc:
            raise DocNotFoundError
        return doc_obj(**doc)