Python boto3.resource() Examples

def __init__(self, aws_access_key_id, aws_secret_access_key,
                 aws_region, in_queue=None, out_queue=None):
        """Set up SQS connections.

        :param aws_access_key_id: AWS access key ID.
        :param aws_secret_access_key: AWS secret access key.
        :param aws_region: AWS region string.
        :param in_queue: Optional input queue name.
        :param out_queue: Optional output queue name.
        """
        self.in_queue = None
        self.out_queue = None

        if in_queue:
            resource = boto3.resource('sqs', region_name=aws_region,
                                      aws_access_key_id=aws_access_key_id,
                                      aws_secret_access_key=aws_secret_access_key)
            self.in_queue = resource.get_queue_by_name(QueueName=in_queue)

        if out_queue:
            client = boto3.client('sqs', region_name=aws_region,
                                  aws_access_key_id=aws_access_key_id,
                                  aws_secret_access_key=aws_secret_access_key)
            self.out_queue = client.get_queue_url(QueueName=out_queue)['QueueUrl'] 

def audit_2_3(self):
        """2.3 Ensure the S3 bucket CloudTrail logs to is not publicly accessible (Scored)"""
        raise NotImplementedError()
        import boto3
        s3 = boto3.session.Session(region_name="us-east-1").resource("s3")
        # s3 = boto3.resource("s3")
        # for trail in self.trails:
        #    for grant in s3.Bucket(trail["S3BucketName"]).Acl().grants:
        #    print(s3.Bucket(trail["S3BucketName"]).Policy().policy)
        for bucket in s3.buckets.all():
            print(bucket)
            try:
                print("    Policy:", bucket.Policy().policy)
            except Exception:
                pass
            for grant in bucket.Acl().grants:
                try:
                    print("    Grant:", grant)
                except Exception:
                    pass 

def assert_alarm(self, name, pattern, remediate=False):
        logs = clients.logs
        sns = resources.sns
        alarm_ok = False
        for trail in self.trails:
            log_group_name = ARN(trail["CloudWatchLogsLogGroupArn"]).resource.split(":")[1]
            for metric_filter in logs.describe_metric_filters(logGroupName=log_group_name)["metricFilters"]:
                if metric_filter["filterPattern"] == pattern:
                    for alarm in self.alarms:
                        try:
                            self.assertEqual(alarm.metric_name, metric_filter["metricTransformations"][0]["metricName"])
                            self.assertGreater(len(list(sns.Topic(alarm.alarm_actions[0]).subscriptions.all())), 0)
                            alarm_ok = True
                        except Exception:
                            pass
        if remediate and not alarm_ok:
            self.ensure_alarm(name=name,
                              pattern=pattern,
                              log_group_name=log_group_name)
            alarm_ok = True
        self.assertTrue(alarm_ok) 

def load_configurations(bucket_name, key_name):
    logging.getLogger().debug('[load_configurations] Start')

    try:
        s3 = boto3.resource('s3')
        file_obj = s3.Object(bucket_name, key_name)
        file_content = file_obj.get()['Body'].read()

        global config
        config = json.loads(file_content)

    except Exception as e:
        logging.getLogger().error("[load_configurations] \tError to read config file")
        raise e

    logging.getLogger().debug('[load_configurations] End') 

def delete_ami(ami_id):
    ensure_ami_exists(ami_id)

    print('Deleting ami: {ami_id}'.format(ami_id=ami_id))
    ec2 = boto3.resource('ec2')
    image = ec2.Image(ami_id)

    # retrieve the mappings before deregistering the image
    mappings = image.block_device_mappings
    print('Got these mappings: {mappings}'.format(mappings=mappings))

    # first we deregister the image
    image.deregister()
    print('Image {ami_id} deregistered'.format(ami_id=ami_id))

    snapshot_ids = [block_device_mapping['Ebs']['SnapshotId'] for block_device_mapping in mappings]
    print('Got snapshots {snapshot_ids}'.format(snapshot_ids=snapshot_ids))
    for snapshot_id in snapshot_ids:
        ec2.Snapshot(snapshot_id).delete()
    print('Deleted snaphots: {snapshot_ids}'.format(snapshot_ids=snapshot_ids)) 

def handle_request(self):
        """
        Handles the custom resource request from cloudformation
        :return: 
        """

        start = datetime.now()

        self._logger.info("Cloudformation request is {}", safe_json(self._event, indent=2))

        try:
            result = CustomResource.handle_request(self)

            return safe_dict({
                "result": result,
                "datetime": datetime.now().isoformat(),
                "running-time": (datetime.now() - start).total_seconds()
            })
        except Exception as ex:
            self._logger.error(ERR_HANDLING_SETUP_REQUEST, ex, full_stack())
            raise ex

        finally:
            self._logger.flush() 

def _create_request(self):
        """
        Handles create request from cloudformation custom resource
        :return: 
        """

        try:
            self._setup()
            self.physical_resource_id = self.__class__.__name__.lower()
            if allow_send_metrics():
                self._send_create_metrics()
            return True

        except Exception as ex:
            self.response["Reason"] = str(ex)
            return False 

def _delete_request(self):
        """
        Handles delete request from cloudformation custom resource
        :return: 
        """

        try:
            self.delete_templates()
            self.delete_external_task_config_stacks()
            if allow_send_metrics():
                self._send_delete_metrics()
            return True

        except Exception as ex:
            self.response["Reason"] = str(ex)
            return False 

def updateVersions(region="us-east-1", table="credential-store"):
    '''
    do a full-table scan of the credential-store,
    and update the version format of every credential if it is an integer
    '''
    dynamodb = boto3.resource('dynamodb', region_name=region)
    secrets = dynamodb.Table(table)

    response = secrets.scan(ProjectionExpression="#N, version, #K, contents, hmac",
                            ExpressionAttributeNames={"#N": "name", "#K": "key"})

    items = response["Items"]

    for old_item in items:
        if isInt(old_item['version']):
            new_item = copy.copy(old_item)
            new_item['version'] = credstash.paddedInt(new_item['version'])
            if new_item['version'] != old_item['version']:
                secrets.put_item(Item=new_item)
                secrets.delete_item(Key={'name': old_item['name'], 'version': old_item['version']})
        else:
            print "Skipping item: %s, %s" % (old_item['name'], old_item['version']) 

def save_directory(self, source_dir, target_name):
        import boto3

        # Stolen from:
        # https://github.com/boto/boto3/issues/358#issuecomment-346093506
        target_name = target_name or os.path.basename(source_dir)
        all_files = []
        for root, dirs, files in os.walk(source_dir):
            all_files += [os.path.join(root, f) for f in files]
        s3_resource = boto3.resource("s3")

        for filename in all_files:
            file_location = s3_join(
                self.dir, target_name, os.path.relpath(filename, source_dir)
            )
            s3_resource.Object(self.bucket, file_location).put(
                Body=open(filename, "rb")
            ) 

def _delete_lambda_role(self, name):
        iam = boto3.resource('iam')

        role_name = self._role_name(name)
        policy_name = self._policy_name(name)

        role_policy = iam.RolePolicy(role_name, policy_name)
        role = iam.Role(self._role_name(name))

        # HACK: This 'if thing in things.all()' biz seems like
        # a very inefficient way to check for resource
        # existence...
        if role_policy in role.policies.all():
            role_policy.delete()

        if role in iam.roles.all():
            role.delete() 

def s3_etag(url: str) -> Optional[str]:
    """Check ETag on S3 object."""
    s3_resource = boto3.resource("s3")
    bucket_name, s3_path = split_s3_path(url)
    s3_object = s3_resource.Object(bucket_name, s3_path)
    return s3_object.e_tag 

def s3_get(url: str, temp_file: IO) -> None:
    """Pull a file directly from S3."""
    s3_resource = boto3.resource("s3")
    bucket_name, s3_path = split_s3_path(url)
    s3_resource.Bucket(bucket_name).download_fileobj(s3_path, temp_file) 

def generate_app_log_parser_conf_file(stack_name, error_threshold, block_period, app_access_log_bucket, overwrite):
    logging.getLogger().debug("[generate_app_log_parser_conf_file] Start")

    local_file = '/tmp/' + stack_name + '-app_log_conf_LOCAL.json'
    remote_file = stack_name + '-app_log_conf.json'
    default_conf = {
        'general': {
            'errorThreshold': error_threshold,
            'blockPeriod': block_period,
            'errorCodes': ['400', '401', '403', '404', '405']
        },
        'uriList': {
        }
    }

    if not overwrite:
        try:
            s3 = boto3.resource('s3')
            file_obj = s3.Object(app_access_log_bucket, remote_file)
            file_content = file_obj.get()['Body'].read()
            remote_conf = json.loads(file_content)

            if 'general' in remote_conf and 'errorCodes' in remote_conf['general']:
                default_conf['general']['errorCodes'] = remote_conf['general']['errorCodes']

            if 'uriList' in remote_conf:
                default_conf['uriList'] = remote_conf['uriList']

        except Exception as e:
            logging.getLogger().debug("[generate_app_log_parser_conf_file] \tFailed to merge existing conf file data.")
            logging.getLogger().debug(e)

    with open(local_file, 'w') as outfile:
        json.dump(default_conf, outfile)

    s3_client = boto3.client('s3')
    s3_client.upload_file(local_file, app_access_log_bucket, remote_file, ExtraArgs={'ContentType': "application/json"})

    logging.getLogger().debug("[generate_app_log_parser_conf_file] End") 

def __init__(self, queue_name, bucket_name, function_name, cron_schedule='rate(1 day)'):
        self._queue_name = queue_name
        self._bucket_name = bucket_name
        self._function_name = function_name
        self._cron_schedule = cron_schedule,
        self._s3 = boto.resource('s3')
        self._sqs = boto.resource('sqs')
        self._lambda_client = boto.client('lambda')
        self._iam_client = boto.client('iam')
        self._queue = None
        self._bucket = None 

def terminate_instance_handler(event, context):
    ec2 = boto3.resource('ec2')
    ec2.Instance(event['instance_id']).terminate()

    return event 

def is_ami_available(ami_id):
    ensure_ami_exists(ami_id)

    ec2 = boto3.resource('ec2')
    image = ec2.Image(ami_id)
    ami_state = image.state
    print('AMI {ami_id} is in state {ami_state}'.format(ami_id=ami_id, ami_state=ami_state))
    return ami_state == "available" 

def create_instance(instance_params):
    ec2 = boto3.resource('ec2')

    for forbidden_param in ['MaxCount', 'MinCount', 'DryRun']:
        if forbidden_param in instance_params:
            del instance_params[forbidden_param]

    # cast Volume sizes to int
    for idx, block_device_mapping in enumerate(instance_params.get('BlockDeviceMappings', [])):
        instance_params['BlockDeviceMappings'][idx]['Ebs']['VolumeSize'] = int(
            instance_params['BlockDeviceMappings'][idx]['Ebs']['VolumeSize']
        )

    instance_id = ec2.create_instances(
        MinCount=1,
        MaxCount=1,
        **instance_params,
    )[0].id

    # just make sure the instance exists before adding tags
    boto3.client('ec2').get_waiter('instance_exists').wait(
        InstanceIds=[instance_id],
    )

    # now we can add tags
    ec2.create_tags(
        Resources=[instance_id],
        Tags=[
            {
                'Key': "UserDataFinished",  # This tag will be set to "true" when the User Data finishes executing
                'Value': 'false'
            },
        ]
    )

    return instance_id 

def user_data_is_completed(instance_id):
    instance = boto3.resource('ec2').Instance(instance_id)
    tags = instance.tags
    print('instance tags:', tags)
    user_data_finished_tag = next(filter(lambda tag: tag['Key'] == 'UserDataFinished', tags))
    return user_data_finished_tag['Value'] == 'true' 

def ensure_ami_with_name_does_not_exist(image_name):
    resource = boto3.resource('ec2')
    images = resource.images.filter(
        Filters=[
            {
                'Name': 'name',
                'Values': [
                    image_name
                ]
            },
        ],
    )
    if len(list(images)) > 0:
        raise AlreadyExistingAMIError(f'AMI name {image_name} is already in use') 

def config_table(self):
        """
        Returns the configuration table
        :return: the configuration table
        """
        table_name = os.getenv(configuration.ENV_CONFIG_TABLE)

        table = boto3.resource("dynamodb").Table(table_name)
        boto_retry.add_retry_methods_to_resource(table, ["scan", "get_item", "delete_item", "put_item"], context=self._context)

        return table 

def config_table_exists():
        tablename = os.environ[configuration.ENV_CONFIG_TABLE]
        for t in boto3.resource("dynamodb").tables.all():
            if t.table_name == tablename:
                return True
        return False 

def validate_tagfilter(tag_filter, action_name):
        """
        Tests if tags are supported by the resources for the action. If this is nit the case then the use of tag filters is
        not possible and an exception is raised
        :param tag_filter: Tag filter value
        :param action_name: Name of the action
        :return: Filter if tags are supported and the filter can be used, otherwise an exception is raised
        """

        if tag_filter is not None:
            tag_filter = tag_filter.strip()

        if tag_filter in ["None", None, ""]:
            return None

        action_properties = actions.get_action_properties(action_name)
        resources = action_properties.get(actions.ACTION_RESOURCES)
        resources_with_tags = services.create_service(action_properties[actions.ACTION_SERVICE]).resources_with_tags

        resource_supports_tags = (resources == "" and len(resources_with_tags) > 0) or resources in resources_with_tags

        # resource does not allow tags, so tag filters can not be used
        if not resource_supports_tags:
            raise_value_error(ERR_NO_TAG_FILTER, action_properties[actions.ACTION_RESOURCES], tag_filter, action_name)

        # destructive actions can deny use of wildcards for tag name
        if not action_properties.get(actions.ACTION_ALLOW_TAGFILTER_WILDCARD, True):
            if "".join([s.strip() for s in tag_filter.split("=")[0:1]]) in ["*", "**", "*="]:
                raise_value_error(ERR_NO_WILDCARDS_TAG_FILTER_ALLOWED, tag_filter, action_name)

        return tag_filter 

def validate_event_scopes(scopes, action_name):

        validated = {}
        # get properties for action for the task and the actions parameter definitions
        action_properties = actions.get_action_properties(action_name)

        action_scopes = action_properties.get(configuration.CONFIG_EVENT_SCOPES, {})
        action_events = action_properties.get(configuration.CONFIG_EVENTS, {})
        for source in scopes:
            if source not in action_scopes or source not in action_events:
                raise_value_error(ERR_EVENT_SCOPE_SOURCE_NOT_HANDLED, source)

            action_detail_event_scopes = action_scopes.get(source, {})
            action_detail_types = action_events.get(source, {})
            for detail_scopes_type in scopes[source]:
                if detail_scopes_type not in action_detail_event_scopes or detail_scopes_type not in action_detail_types:
                    raise_value_error(ERR_EVENT_SCOPE_DETAIL_TYPE_NOT_HANDLED, detail_scopes_type, source)

                action_scope_events = action_detail_event_scopes.get(detail_scopes_type, [])
                action_supported_events = action_detail_types.get(detail_scopes_type, [])
                for event in scopes[source][detail_scopes_type]:
                    if event not in action_scope_events or event not in action_supported_events:
                        raise_value_error(ERR_EVENT_SCOPE_EVENT_NOT_HANDLED, event, source, detail_scopes_type)
                    if action_scope_events[event] not in VALID_EVENT_SCOPES:
                        raise_value_error(ERR_INVALID_EVENT_SCOPE, action_scope_events[event], ",".join(VALID_EVENT_SCOPES))

                # only use values other than default resource value
                scopes_for_detail_type = {s: scopes[source][detail_scopes_type][s] for s in scopes[source][detail_scopes_type] if
                                          scopes[source][detail_scopes_type][s] != handlers.EVENT_SCOPE_RESOURCE}

                if len(scopes_for_detail_type) > 0:
                    if source not in validated:
                        validated[source] = {}
                    validated[source][detail_scopes_type] = scopes_for_detail_type

        return validated 

def __enter__(self):
        """
        Returns itself as the managed resource.
        :return:
        """
        return self 

def is_handling_request(cls, event, _):
        """
        Test if the event is handled by this handler
        :param _:
        :param event: Event to test
        :return: True if the event is an event from cloudformationOpsAutomatorSetupHelper custom resource
        """
        return event.get("StackId") is not None and event.get("ResourceType") == "Custom::OpsAutomatorSetupHelper" 

def _update_request(self):
        """
        Handles update request from cloudformation custom resource
        :return: 
        """

        try:
            self._setup()
            return True

        except Exception as ex:
            self.response["Reason"] = str(ex)
            return False 

def _last_run_table(self):
        """
        Returns table to store last execution time for this handler.
        :return: table to store last execution time for this handler
        """
        if self._table is None:
            self._table = boto3.resource('dynamodb').Table(os.environ[handlers.ENV_LAST_RUN_TABLE])
            add_retry_methods_to_resource(self._table, ["get_item", "update_item"])
        return self._table 

def s3_resources(self):
        if self._s3_resources is None:
            self._s3_resources = boto3.resource("s3")
        return self._s3_resources 

def s3_etag(url: str) -> Optional[str]:
    """Check ETag on S3 object."""
    s3_resource = boto3.resource("s3")
    bucket_name, s3_path = split_s3_path(url)
    s3_object = s3_resource.Object(bucket_name, s3_path)
    return s3_object.e_tag