Python celery.exceptions.Retry() Examples

def test_save_sms_should_go_to_retry_queue_if_database_errors(sample_template, mocker):
    notification = _notification_json(sample_template, "+447234123123")

    expected_exception = SQLAlchemyError()

    mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
    mocker.patch('app.celery.tasks.save_sms.retry', side_effect=Retry)
    mocker.patch('app.notifications.process_notifications.dao_create_notification', side_effect=expected_exception)

    notification_id = uuid.uuid4()

    with pytest.raises(Retry):
        save_sms(
            sample_template.service_id,
            notification_id,
            encryption.encrypt(notification),
        )
    assert provider_tasks.deliver_sms.apply_async.called is False
    tasks.save_sms.retry.assert_called_with(exc=expected_exception, queue="retry-tasks")

    assert Notification.query.count() == 0 

def test_save_email_should_go_to_retry_queue_if_database_errors(sample_email_template, mocker):
    notification = _notification_json(sample_email_template, "test@example.gov.uk")

    expected_exception = SQLAlchemyError()

    mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
    mocker.patch('app.celery.tasks.save_email.retry', side_effect=Retry)
    mocker.patch('app.notifications.process_notifications.dao_create_notification', side_effect=expected_exception)

    notification_id = uuid.uuid4()

    with pytest.raises(Retry):
        save_email(
            sample_email_template.service_id,
            notification_id,
            encryption.encrypt(notification),
        )
    assert not provider_tasks.deliver_email.apply_async.called
    tasks.save_email.retry.assert_called_with(exc=expected_exception, queue="retry-tasks")

    assert Notification.query.count() == 0 

def test_fn_retry_exception(celery_app, memory_exporter):
    @celery_app.task
    def fn_exception():
        raise Retry("Task class is being retried")

    result = fn_exception.apply()

    assert result.failed() is False
    assert "Task class is being retried" in result.traceback

    spans = memory_exporter.get_finished_spans()
    assert len(spans) == 1

    span = spans[0]

    assert span.status.is_ok is True
    assert span.status.canonical_code == StatusCanonicalCode.OK
    assert span.name == "test_celery_functional.fn_exception"
    assert span.attributes.get("celery.action") == "run"
    assert span.attributes.get("celery.state") == "RETRY"
    assert (
        span.attributes.get("celery.task_name")
        == "test_celery_functional.fn_exception"
    )
    assert span.attributes.get("messaging.message_id") == result.task_id 

def git_push_if_needed(remote, branch, cwd=None):
    """
    Push current HEAD to remote branch.

    Return True if push succeeded, False if there was nothing to push.
    Raises a celery Retry exception in case of non-fast-forward push.
    """
    r = call(["git", "diff", "--quiet", "--exit-code", remote + "/" + branch], cwd=cwd)
    if r == 0:
        return False
    try:
        check_call(["git", "push", remote, branch], cwd=cwd, log_error=False)
    except CalledProcessError as e:
        if "non-fast-forward" in e.output:
            raise Retry(
                exc=e,
                message="Retrying because a non-fast-forward git push was attempted.",
            )
        else:
            _logger.error(
                f"command {e.cmd} failed with return code {e.returncode} "
                f"and output:\n{e.output}"
            )
            raise
    return True 

def test_sync_data_to_customer_cold_storage_retry(self, mock_sync, mock_data_export_request):
        """Test that the sync task retries syncer fails with a cold storage error."""
        data_export_object = Mock()
        data_export_object.uuid = fake.uuid4()
        data_export_object.status = APIExportRequest.PENDING

        mock_data_export_request.get.return_value = data_export_object
        mock_sync.return_value.sync_bucket.side_effect = SyncedFileInColdStorageError()
        with self.assertRaises(Retry):
            tasks.sync_data_to_customer(data_export_object.uuid)
        self.assertEquals(data_export_object.status, APIExportRequest.WAITING) 

def test_run_job_dangling_job(
    mocker, now, one_hour_ago, one_hour_ahead, spark_job_with_run_factory
):
    # create a job with a run that started one hour ago and is allowed
    # to run for two hours, so it's not timing out, but it's not quite
    # healthy, too
    spark_job_with_run = spark_job_with_run_factory(
        start_date=one_hour_ahead,
        job_timeout=2,
        run__status=Cluster.STATUS_WAITING,
        run__scheduled_at=one_hour_ago,
    )
    mocker.spy(tasks.run_job, "terminate_and_notify")
    mocker.patch(
        "atmo.clusters.provisioners.ClusterProvisioner.info",
        return_value={
            "creation_datetime": now,
            "ready_datetime": None,
            "end_datetime": None,
            "state": Cluster.STATUS_WAITING,
            "public_dns": None,
        },
    )
    terminate = mocker.patch("atmo.jobs.models.SparkJob.terminate")
    assert not spark_job_with_run.has_finished
    assert not spark_job_with_run.has_timed_out
    assert terminate.call_count == 0

    # tries running again
    with pytest.raises(Retry):
        tasks.run_job(spark_job_with_run.pk)

    assert tasks.run_job.terminate_and_notify.call_count == 0
    assert terminate.call_count == 0 

def test_lock_is_not_acquireable___retry_esception_is_raised(self, pk, location, analysis_settings_path):
        with patch('fasteners.InterProcessLock.acquire', Mock(return_value=False)), \
             patch('src.model_execution_worker.tasks.notify_api_status') as api_notify:
            with self.assertRaises(Retry):
                start_analysis_task(pk, location, analysis_settings_path) 

def test_stack_calibrations_not_enough_images(self, mock_get_instrument, mock_get_calibration_images, mock_maker, setup):
        mock_get_instrument.return_value = self.fake_inst
        mock_get_calibration_images.return_value = [FakeLCOObservationFrame(hdu_list=[FakeCCDData()])]
        with pytest.raises(Retry) as e:
            stack_calibrations(self.min_date, self.max_date, 1, self.frame_type, self.context,
                               [self.fake_blocks_response_json['results'][0]])
        assert e.type is Retry 

def gh_call(func, *args, **kwargs):
    """Intercept GitHub call to wait when the API rate limit is reached."""
    try:
        return func(*args, **kwargs)
    except github3.exceptions.ForbiddenError as e:
        if not e.response.headers.get("X-RateLimit-Remaining", 1):
            raise Retry(
                message="Retry task after rate limit reset",
                exc=e,
                when=e.response.headers.get("X-RateLimit-Reset"),
            )
        raise 

def test_git_push_if_needed(git_clone):
    assert not git_push_if_needed("origin", "master", cwd=git_clone)
    afile = git_clone / "afile"
    with afile.open("w"):
        pass
    subprocess.check_call(["git", "add", "afile"], cwd=git_clone)
    subprocess.check_call(["git", "commit", "-m", "add afile"], cwd=git_clone)
    assert git_push_if_needed("origin", "master", cwd=git_clone)
    assert not git_push_if_needed("origin", "master", cwd=git_clone)
    subprocess.check_call(["git", "reset", "--hard", "HEAD^"], cwd=git_clone)
    with pytest.raises(Retry):
        git_push_if_needed("origin", "master", cwd=git_clone) 

def process_ses_results(self, response):
    try:
        ses_message = json.loads(response['Message'])
        notification_type = ses_message['notificationType']

        if notification_type == 'Bounce':
            notification_type = determine_notification_bounce_type(notification_type, ses_message)
        elif notification_type == 'Complaint':
            _check_and_queue_complaint_callback_task(*handle_complaint(ses_message))
            return True

        aws_response_dict = get_aws_responses(notification_type)

        notification_status = aws_response_dict['notification_status']
        reference = ses_message['mail']['messageId']

        try:
            notification = notifications_dao.dao_get_notification_or_history_by_reference(reference=reference)
        except NoResultFound:
            message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None)
            if datetime.utcnow() - message_time < timedelta(minutes=5):
                current_app.logger.info(
                    f"notification not found for reference: {reference} (update to {notification_status}). "
                    f"Callback may have arrived before notification was persisted to the DB. Adding task to retry queue"
                )
                self.retry(queue=QueueNames.RETRY)
            else:
                current_app.logger.warning(
                    f"notification not found for reference: {reference} (update to {notification_status})"
                )
            return

        if notification.status not in [NOTIFICATION_SENDING, NOTIFICATION_PENDING]:
            notifications_dao._duplicate_update_warning(
                notification=notification,
                status=notification_status
            )
            return
        else:
            notifications_dao.dao_update_notifications_by_reference(
                references=[reference],
                update_dict={'status': notification_status}
            )

        statsd_client.incr('callback.ses.{}'.format(notification_status))

        if notification.sent_at:
            statsd_client.timing_with_dates('callback.ses.elapsed-time', datetime.utcnow(), notification.sent_at)

        _check_and_queue_callback_task(notification)

        return True

    except Retry:
        raise

    except Exception as e:
        current_app.logger.exception('Error processing SES results: {}'.format(type(e)))
        self.retry(queue=QueueNames.RETRY)