Python celery.exceptions.MaxRetriesExceededError() Examples

def check_transaction_status_in_pool(self, check_sent_to_pool_result):

    transaction_hash, blockchain_transaction_ids = check_sent_to_pool_result

    try:
        success = blockchain_processor.check_transaction_status_in_pool(transaction_hash, blockchain_transaction_ids)

        if not success:
            self.retry(countdown=BITCOIN_CHECK_TRANSACTION_BASE_TIME * 2 ** self.request.retries)

    except MaxRetriesExceededError:

        submitted_date = datetime.utcnow()

        for transaction_id in blockchain_transaction_ids:
            response = blockchain_processor.put_blockchain_result_to_app({
                'transaction_id': transaction_id,
                'status': 'FAILED',
                'message': 'Timeout in pool',
                'transaction_type': 'transfer',
                'submitted_date': submitted_date.isoformat()
            })
    return True 

def test_get_pdf_for_templated_letter_sets_technical_failure_max_retries(mocker, sample_letter_notification):
    mock_celery = mocker.patch('app.celery.letters_pdf_tasks.notify_celery.send_task', side_effect=Exception())
    mocker.patch('app.celery.letters_pdf_tasks.get_letter_pdf_filename', return_value='LETTER.PDF')
    mock_retry = mocker.patch(
        'app.celery.letters_pdf_tasks.get_pdf_for_templated_letter.retry', side_effect=MaxRetriesExceededError)
    mock_update_noti = mocker.patch('app.celery.letters_pdf_tasks.update_notification_status_by_id')

    with pytest.raises(NotificationTechnicalFailureException) as e:
        get_pdf_for_templated_letter(sample_letter_notification.id)

    assert e.value.args[0] == f"RETRY FAILED: Max retries reached. " \
        f"The task create-letter-pdf failed for notification id {sample_letter_notification.id}. " \
        f"Notification has been updated to technical-failure"
    assert mock_celery.called
    assert mock_retry.called
    mock_update_noti.assert_called_once_with(sample_letter_notification.id, 'technical-failure') 

def test_process_sanitised_letter_puts_letter_into_technical_failure_if_max_retries_exceeded(
    mocker,
    sample_letter_notification,
):
    mocker.patch('app.celery.letters_pdf_tasks.update_letter_pdf_status', side_effect=Exception())
    mocker.patch('app.celery.letters_pdf_tasks.process_sanitised_letter.retry', side_effect=MaxRetriesExceededError())

    sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK
    encrypted_data = encryption.encrypt({
        'page_count': 2,
        'message': None,
        'invalid_pages': None,
        'validation_status': 'passed',
        'filename': 'NOTIFY.{}'.format(sample_letter_notification.reference),
        'notification_id': str(sample_letter_notification.id),
        'address': None
    })

    with pytest.raises(NotificationTechnicalFailureException):
        process_sanitised_letter(encrypted_data)

    assert sample_letter_notification.status == NOTIFICATION_TECHNICAL_FAILURE 

def check_whether_transaction_sent_to_pool(self, submit_result):
    transaction_hash, credit_transfer_ids = submit_result

    try:
        check_sent_to_pool_result = (
            blockchain_processor.check_whether_transaction_sent_to_pool(transaction_hash, credit_transfer_ids)
        )

        if not check_sent_to_pool_result:
            raise Exception("Not found in pool")
        else:
            return check_sent_to_pool_result

    except Exception as e:
        # This is so we can log the underlying error rather than just getting a MaxRetriesExceededError
        if self.request.retries < self.max_retries - 1:
            self.retry(countdown=2 * 2 ** self.request.retries)
        else:
            submitted_date = datetime.utcnow()

            for credit_transfer_id in credit_transfer_ids:
                response = blockchain_processor.post_blockchain_result_to_app({
                    'credit_transfer_id': credit_transfer_id,
                    'status': 'FAILED',
                    'message': 'Failed to add to pool: ' + str(e),
                    'transaction_type': 'transfer',
                    'submitted_date': submitted_date.isoformat(),
                    'force_transaction_creation': True
                })

            raise e
    return True 

def test_should_go_into_technical_error_if_exceeds_retries_on_deliver_sms_task(sample_notification, mocker):
    mocker.patch('app.delivery.send_to_providers.send_sms_to_provider', side_effect=Exception("EXPECTED"))
    mocker.patch('app.celery.provider_tasks.deliver_sms.retry', side_effect=MaxRetriesExceededError())

    with pytest.raises(NotificationTechnicalFailureException) as e:
        deliver_sms(sample_notification.id)
    assert str(sample_notification.id) in str(e.value)

    provider_tasks.deliver_sms.retry.assert_called_with(queue="retry-tasks", countdown=0)

    assert sample_notification.status == 'technical-failure' 

def test_should_go_into_technical_error_if_exceeds_retries_on_deliver_email_task(sample_notification, mocker):
    mocker.patch('app.delivery.send_to_providers.send_email_to_provider', side_effect=Exception("EXPECTED"))
    mocker.patch('app.celery.provider_tasks.deliver_email.retry', side_effect=MaxRetriesExceededError())

    with pytest.raises(NotificationTechnicalFailureException) as e:
        deliver_email(sample_notification.id)
    assert str(sample_notification.id) in str(e.value)

    provider_tasks.deliver_email.retry.assert_called_with(queue="retry-tasks")
    assert sample_notification.status == 'technical-failure' 

def test_sanitise_letter_puts_letter_into_technical_failure_if_max_retries_exceeded(sample_letter_notification, mocker):
    mocker.patch('app.celery.letters_pdf_tasks.notify_celery.send_task', side_effect=Exception())
    mocker.patch('app.celery.letters_pdf_tasks.sanitise_letter.retry', side_effect=MaxRetriesExceededError())

    filename = 'NOTIFY.{}'.format(sample_letter_notification.reference)
    sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK

    with pytest.raises(NotificationTechnicalFailureException):
        sanitise_letter(filename)

    assert sample_letter_notification.status == NOTIFICATION_TECHNICAL_FAILURE 

def process_map_updates(self):
    if self.request.called_directly:
        logger.info('Processing map updates by direct command...')
    else:
        logger.info('Processing map updates...')

    from c3nav.mapdata.models import MapUpdate
    try:
        try:
            updates = MapUpdate.process_updates()
        except MapUpdate.ProcessUpdatesAlreadyRunning:
            if self.request.called_directly:
                raise
            logger.info('Processing is already running, retrying in 30 seconds.')
            raise self.retry(countdown=30)
        except Exception:
            cache.set('mapdata:last_process_updates_run', (int(time.time()), False), None)
            raise
        else:
            cache.set('mapdata:last_process_updates_run', (int(time.time()), True), None)
    except MaxRetriesExceededError:
        logger.info('Cannot retry, retries exceeded. Exiting.')
        return

    if updates:
        print()

    logger.info(ungettext_lazy('%d map update processed.', '%d map updates processed.', len(updates)) % len(updates))

    if updates:
        logger.info(_('Last processed update: %(date)s (#%(id)d)') % {
            'date': date_format(updates[-1].datetime, 'DATETIME_FORMAT'),
            'id': updates[-1].pk,
        }) 

def create_transaction_response(self, previous_result, credit_transfer_id):

    def transaction_response_countdown():
        t = lambda retries: ETH_CHECK_TRANSACTION_BASE_TIME*2**retries

        # If the system has been longer than the max retry period
        if previous_result:
            submitted_at = datetime.strptime(previous_result['submitted_date'], "%Y-%m-%d %H:%M:%S.%f")
            if (datetime.utcnow() - submitted_at).total_seconds() > ETH_CHECK_TRANSACTION_RETRIES_TIME_LIMIT:
                if self.request.retries != self.max_retries:
                    self.request.retries = self.max_retries - 1

                return 0

        return t(self.request.retries)

    try:
        try:
            result = blockchain_processor.create_transaction_response(previous_result['transaction_hash'])

        except Exception as e:
            print(e)
            self.retry(countdown=transaction_response_countdown())

        else:
            result = {**previous_result, **result, 'credit_transfer_id': credit_transfer_id}

            blockchain_processor.send_blockchain_result_to_app(result)

            if result['status'] == 'PENDING':
                self.retry(countdown=transaction_response_countdown())

            return result

    except MaxRetriesExceededError as e:

        result = {**previous_result,
                  'status': 'FAILED',
                  'message': 'timeout',
                  'credit_transfer_id': credit_transfer_id}

        blockchain_processor.send_blockchain_result_to_app(result)

        raise e
    return True 

def sync_data_to_customer(dump_request_uuid):
    """
    Scheduled task to sync normalized data to our customers S3 bucket.

    If the sync request raises SyncedFileInColdStorageError, this task
    will automatically retry in a set amount of time. This time is to give
    the storage solution time to retrieve a file from cold storage.
    This task will retry 5 times, and then fail.

    """
    dump_request = DataExportRequest.objects.get(uuid=dump_request_uuid)
    dump_request.status = DataExportRequest.PROCESSING
    dump_request.save()

    try:
        syncer = AwsS3Syncer(settings.S3_BUCKET_NAME)
        syncer.sync_bucket(
            dump_request.created_by.customer.schema_name,
            dump_request.bucket_name,
            (dump_request.start_date, dump_request.end_date),
        )
    except ClientError:
        LOG.exception(
            f"Encountered an error while processing DataExportRequest "
            f"{dump_request.uuid}, for {dump_request.created_by}."
        )
        dump_request.status = DataExportRequest.ERROR
        dump_request.save()
        return
    except SyncedFileInColdStorageError:
        LOG.info(
            f"One of the requested files is currently in cold storage for "
            f"DataExportRequest {dump_request.uuid}. This task will automatically retry."
        )
        dump_request.status = DataExportRequest.WAITING
        dump_request.save()
        try:
            raise sync_data_to_customer.retry(countdown=10, max_retries=5)
        except MaxRetriesExceededError:
            LOG.exception(
                f"Max retires exceeded for restoring a file in cold storage for "
                f"DataExportRequest {dump_request.uuid}, for {dump_request.created_by}."
            )
            dump_request.status = DataExportRequest.ERROR
            dump_request.save()
            return
    dump_request.status = DataExportRequest.COMPLETE
    dump_request.save()