Python celery.shared_task() Examples

def task(*d_args, **d_kwargs):
    # behaves like @task, but emails about exceptions.
    def real_decorator(f):
        @shared_task(*d_args, **d_kwargs)
        @wraps(f)
        def wrapper(*f_args, **f_kwargs):
            # try the task; email any exceptions we get
            try:
                res = f(*f_args, **f_kwargs)
            except Exception as e:
                # email admins and re-raise
                exc_type, exc_value, exc_traceback = sys.exc_info()
                subject = 'task failure in %s.%s' % (f.__module__, f.__name__)
                msg = 'The task %s.%s failed:\n\n%s' % (f.__module__, f.__name__,
                        '\n'.join(traceback.format_exception(exc_type, exc_value, exc_traceback)))
                mail_admins(subject=subject, message=msg, fail_silently=True)
                raise

            return res
        return wrapper
    return real_decorator 

def test_shared_task(celery_app, memory_exporter):
    """Ensure Django Shared Task are supported"""

    @celery.shared_task
    def add(x, y):
        return x + y

    result = add.apply([2, 2])
    assert result.result == 4

    spans = memory_exporter.get_finished_spans()
    assert len(spans) == 1

    span = spans[0]

    assert span.status.is_ok is True
    assert span.name == "test_celery_functional.add"
    assert (
        span.attributes.get("celery.task_name") == "test_celery_functional.add"
    )
    assert span.attributes.get("celery.action") == "run"
    assert span.attributes.get("celery.state") == "SUCCESS"
    assert span.attributes.get("messaging.message_id") == result.task_id 

def shared_task_conditional(**kwargs):  # pragma: no cover
    """Decorator to optionally disable celery tests."""
    def decorator(func):
        if settings.NO_CELERY:
            setattr(func, 'delay', func)
            return func

        # Get the real decorator
        dec = shared_task(func, **kwargs)

        # Create a stub to apply a countdown and run
        def patched(*args, **kwargs):
            return dec.apply_async(
                args=args, kwargs=kwargs, countdown=settings.CELERY_DELAY)

        # Substitute the delay method
        dec.delay = patched
        return dec

    return decorator 

def shared_task(task):  # noqa: D103, pylint: disable=missing-docstring
        return task 

def task(self):
        return shared_task(**self.config)(self._run_task)


# Define tasks so that Celery can discovery them 

def shared_task(func):
        def wrapper(*args, **kwargs):
            return func(*args, **kwargs)
        return wrapper 

def _create_db_asset(fcn, model, args, key):
    if getattr(settings, 'LEDGER_SYNC_ENABLED', True):
        return __create_db_asset(model, fcn, args, key, sync=True)
    else:
        shared_task(__create_db_asset)(model, fcn, args, key, sync=False)
        return {'message': _MESSAGE} 

def _create_db_assets(fcn, model, args, keys):
    if getattr(settings, 'LEDGER_SYNC_ENABLED', True):
        return __create_db_assets(model, fcn, args, keys, sync=True)
    else:
        shared_task(__create_db_asset)(model, fcn, args, keys, sync=False)
        return {'message': _MESSAGE} 

def __create_asset(fcn, args, sync=False, **extra_kwargs):
    # create a wrapper as it seems the shared_task decorator from celery is not
    # compatible with our retry decorator on the invoke_ledger function
    return invoke_ledger(fcn=fcn, args=args, sync=sync, **extra_kwargs) 

def _create_asset(fcn, args, **extra_kwargs):
    if getattr(settings, 'LEDGER_SYNC_ENABLED', True):
        return __create_asset(fcn, args=args, sync=True, **extra_kwargs)
    else:
        shared_task(__create_asset)(fcn, args=args, sync=False, **extra_kwargs)
        return {'message': _MESSAGE} 

def configure_worker(options={}, **kwargs):

    if 'queues' not in options:
        return
    if CORRESPONDING_QUEUE not in options['queues'].split(','):
        return
    print('### STARTING UP A TREE BUILDER WORKER ###')

    global retroTransformer
    # Instantiate and load retro transformer
    retroTransformer = RetroTransformer(celery=True)
    retroTransformer.load(chiral=False)

    print('### TREE BUILDER WORKER STARTED UP ###')


# ONLY ONE WORKER TYPE HAS THIS FUNCTION EXPOSED - MAKE IT THE CHIRAL ONE
# @shared_task
# def fast_filter_check(*args, **kwargs):
#     '''Wrapper for fast filter check, since these workers will 
#     have it initialized. Best way to allow independent queries'''
#     global retroTransformer
#     if not retroTransformer.fast_filter:
#         from makeit.synthetic.evaluation.fast_filter import FastFilterScorer
#         retroTransformer.fast_filter = FastFilterScorer()
#         retroTransformer.fast_filter.load(model_path=gc.FAST_FILTER_MODEL['trained_model_path'])
#     return retroTransformer.fast_filter.evaluate(*args, **kwargs) 

def shared_task(func):
        # Dummy decorator so that we can use the decorator wether celery is installed or not

        # We do not yet need this, but might come in handy in the future:
        #func.delay = lambda *a, **kw: func(*a, **kw)
        #func.apply_async = lambda *a, **kw: func(*a, **kw)
        return func 

def mutex_task(task_id_template=None, **shared_task_kwargs):
    """ Wraps a task that must be executed only once.

    :param task_id_template: String that makes unique task IDs from passed args
        (If omitted, we just use the function name)
    :param shared_task_kwargs: Passed through to `shared_task`
    """

    def decorator(func):
        signature = inspect.signature(func)

        @shared_task(**shared_task_kwargs, bind=True)
        @wraps(func)
        def wrapped_task(self, *task_args, **task_kwargs):
            if task_id_template:
                passed_args = signature.bind(*task_args, **task_kwargs)
                passed_args.apply_defaults()
                task_identifier = task_id_template.format_map(passed_args.arguments)
            else:
                task_identifier = func.__name__

            with exclusive_lock(task_identifier) as has_lock:
                if has_lock:
                    return func(*task_args, **task_kwargs)
                logger.debug("Other worker already processing %s", task_identifier)
            return None

        return wrapped_task

    return decorator