Python celery.uuid() Examples

def costruisci_e_accoda(oggetto=None, modello=None, corpo=None, mittente=None,
                            destinatari=None, allegati=None, **kwargs):
        """
        Scorciatoia per costruire rapidamente un messaggio di posta e inviarlo alla coda celery.
        :param oggetto: Oggetto deltilizzare per l'invio.
        :param modello: Modello da utilizzare per l'invio.
        :param corpo: Sostituzioni da fare nel modello. Dizionario {nome: valore}
        :param mittente: Mittente del messaggio. None per Notifiche da Gaia.
        :param destinatari: Un elenco di destinatari (oggetti Persona).
        :param allegati: Allegati messaggio
        :return: Un oggetto Messaggio accodato.
        """

        msg = Messaggio.costruisci_email(oggetto=oggetto, modello=modello,
                                         corpo=corpo, mittente=mittente,
                                         destinatari=destinatari,
                                         allegati=allegati, **kwargs)

        # Crea un ID per il task Celery
        msg.task_id = uuid()
        msg.save()

        invia_mail.apply_async((msg.pk,), task_id=msg.task_id)

        return msg 

def test_backend__pickle_serialization(self):
        self.app.conf.result_serializer = 'pickle'
        self.app.conf.accept_content = {'pickle', 'json'}
        self.b = DatabaseBackend(app=self.app)

        tid2 = uuid()
        result = {'foo': 'baz', 'bar': SomeClass(12345)}
        self.b.mark_as_done(tid2, result)
        # is serialized properly.
        rindb = self.b.get_result(tid2)
        assert rindb.get('foo') == 'baz'
        assert rindb.get('bar').data == 12345

        tid3 = uuid()
        try:
            raise KeyError('foo')
        except KeyError as exception:
            self.b.mark_as_failure(tid3, exception)

        assert self.b.get_status(tid3) == states.FAILURE
        assert isinstance(self.b.get_result(tid3), KeyError) 

def xxx_backend(self):
        tid = uuid()

        assert self.b.get_status(tid) == states.PENDING
        assert self.b.get_result(tid) is None

        self.b.mark_as_done(tid, 42)
        assert self.b.get_status(tid) == states.SUCCESS
        assert self.b.get_result(tid) == 42

        tid2 = uuid()
        try:
            raise KeyError('foo')
        except KeyError as exception:
            self.b.mark_as_failure(tid2, exception)

        assert self.b.get_status(tid2) == states.FAILURE
        assert isinstance(self.b.get_result(tid2), KeyError) 

def test_backend__pickle_serialization(self):
        self.app.conf.result_serializer = 'pickle'
        self.app.conf.accept_content = {'pickle', 'json'}
        self.b = DatabaseBackend(app=self.app)

        tid2 = uuid()
        result = {'foo': 'baz', 'bar': SomeClass(12345)}
        self.b.mark_as_done(tid2, result)
        # is serialized properly.
        rindb = self.b.get_result(tid2)
        assert rindb.get('foo') == 'baz'
        assert rindb.get('bar').data == 12345

        tid3 = uuid()
        try:
            raise KeyError('foo')
        except KeyError as exception:
            self.b.mark_as_failure(tid3, exception)

        assert self.b.get_status(tid3) == states.FAILURE
        assert isinstance(self.b.get_result(tid3), KeyError) 

def xxx_backend(self):
        tid = uuid()

        assert self.b.get_status(tid) == states.PENDING
        assert self.b.get_result(tid) is None

        self.b.mark_as_done(tid, 42)
        assert self.b.get_status(tid) == states.SUCCESS
        assert self.b.get_result(tid) == 42

        tid2 = uuid()
        try:
            raise KeyError('foo')
        except KeyError as exception:
            self.b.mark_as_failure(tid2, exception)

        assert self.b.get_status(tid2) == states.FAILURE
        assert isinstance(self.b.get_result(tid2), KeyError) 

def make(self):
        # Verifica appartenenza dell'elenco richiesto.
        if not self.may_be_downloaded_async:
            return self.download()  # scarica direttamente

        # Create new record in DB
        task_uuid = uuid()
        report_db = ReportElenco(report_type=self.elenco_report_type,
                                 task_id=task_uuid,
                                 user=self.request.user.persona)
        report_db.save()

        # Partire celery task e reindirizza user sulla pagina "Report Elenco"
        task = generate_elenco.apply_async(
            (self.celery_params, report_db.id),
            task_id=task_uuid)

        # Can be that the report is generated immediately, wait a bit
        # and refresh db-record to verify
        from time import sleep
        sleep(2.5)
        report_db.refresh_from_db()

        if report_db.is_ready and report_db.file:
            # If the report is ready, download it without redirect user
            return report_db.download()
        else:
            response = redirect(reverse('ufficio_soci:elenchi_richiesti_download'))
            messages.success(self.request, 'Attendi la generazione del report richiesto.')
            return response 

def task_pcap_create():
    """Endpoint for network/pcap analysis task."""
    if 'pcap' not in request.files:
        # no pcap file
        res = ErrorAPIResponse(2010).to_dict()
        return jsonify(res), 400

    pcap_file = request.files['pcap']

    if pcap_file.filename == '':
        # noname file
        res = ErrorAPIResponse(2011).to_dict()
        return jsonify(res), 400

    # get pretty print parameter
    pretty = False
    if 'pretty' in request.form:
        pretty = request.form['pretty']
        if pretty not in ('true', 'false'):
            res = ErrorAPIResponse(2000).to_dict()
            return jsonify(res), 400

    task_id = uuid()

    # prepare directory and save pcap
    os.mkdir(f'{storage_path}/{task_id}')
    pcap_path = f'{storage_path}/{task_id}/{pcap_file.filename}'
    pcap_file.save(pcap_path)

    # run pcap analysis
    args = (pcap_path,)
    kwargs = {'pretty': pretty}
    tasks.pcap_analysis.apply_async(args, kwargs, task_id=task_id)

    res = {
        'task_id': task_id
    }
    return jsonify(res) 

def send_via_mail(self, redirect_url, target):
        task = send_mail.apply_async(args=(self.get_user_pk, target), task_id=uuid())

        # messages.add_message(self.request, messages.INFO, self.CELERY_TASK_PREFIX+task.id)
        return redirect_url 

def create_task_result(self):
        id = uuid()
        taskmeta, created = TaskResult.objects.get_or_create(task_id=id)
        return taskmeta 

def test_mark_as_failure(self):
        einfo = None
        tid3 = uuid()
        try:
            raise KeyError('foo')
        except KeyError as exception:
            einfo = ExceptionInfo(sys.exc_info())
            self.b.mark_as_failure(tid3, exception, traceback=einfo.traceback)
        assert self.b.get_status(tid3) == states.FAILURE
        assert isinstance(self.b.get_result(tid3), KeyError)
        assert self.b.get_traceback(tid3) == einfo.traceback 

def test_is_pickled(self):
        tid2 = uuid()
        result = {'foo': 'baz', 'bar': SomeClass(12345)}
        self.b.mark_as_done(tid2, result)
        # is serialized properly.
        rindb = self.b.get_result(tid2)
        assert rindb.get('foo') == 'baz'
        assert rindb.get('bar').data == 12345 

def test_forget(self):
        tid = uuid()
        self.b.mark_as_done(tid, {'foo': 'bar'})
        assert self.b.get_result(tid).get('foo') == 'bar'
        self.b.forget(tid)
        assert tid not in self.b._cache
        assert self.b.get_result(tid) is None 

def test_mark_as_done(self):
        tid = uuid()

        assert self.b.get_status(tid) == states.PENDING
        assert self.b.get_result(tid) is None

        self.b.mark_as_done(tid, 42)
        assert self.b.get_status(tid) == states.SUCCESS
        assert self.b.get_result(tid) == 42 

def test_forget(self):
        tid = uuid()
        self.b.mark_as_done(tid, {'foo': 'bar'})
        x = self.app.AsyncResult(tid)
        assert x.result.get('foo') == 'bar'
        x.forget()
        if celery.VERSION[0:3] == (3, 1, 10):
            # bug in 3.1.10 means result did not clear cache after forget.
            x._cache = None
        assert x.result is None 

def create_task_result(self):
        id = uuid()
        taskmeta, created = TaskResult.objects.get_or_create(task_id=id)
        return taskmeta 

def test_mark_as_failure(self):
        einfo = None
        tid3 = uuid()
        try:
            raise KeyError('foo')
        except KeyError as exception:
            einfo = ExceptionInfo(sys.exc_info())
            self.b.mark_as_failure(tid3, exception, traceback=einfo.traceback)
        assert self.b.get_status(tid3) == states.FAILURE
        assert isinstance(self.b.get_result(tid3), KeyError)
        assert self.b.get_traceback(tid3) == einfo.traceback 

def test_is_pickled(self):
        tid2 = uuid()
        result = {'foo': 'baz', 'bar': SomeClass(12345)}
        self.b.mark_as_done(tid2, result)
        # is serialized properly.
        rindb = self.b.get_result(tid2)
        assert rindb.get('foo') == 'baz'
        assert rindb.get('bar').data == 12345 

def test_forget(self):
        tid = uuid()
        self.b.mark_as_done(tid, {'foo': 'bar'})
        assert self.b.get_result(tid).get('foo') == 'bar'
        self.b.forget(tid)
        assert tid not in self.b._cache
        assert self.b.get_result(tid) is None 

def test_mark_as_done(self):
        tid = uuid()

        assert self.b.get_status(tid) == states.PENDING
        assert self.b.get_result(tid) is None

        self.b.mark_as_done(tid, 42)
        assert self.b.get_status(tid) == states.SUCCESS
        assert self.b.get_result(tid) == 42 

def test_forget(self):
        tid = uuid()
        self.b.mark_as_done(tid, {'foo': 'bar'})
        x = self.app.AsyncResult(tid)
        assert x.result.get('foo') == 'bar'
        x.forget()
        if celery.VERSION[0:3] == (3, 1, 10):
            # bug in 3.1.10 means result did not clear cache after forget.
            x._cache = None
        assert x.result is None 

def handleCreateRelease(token, package, title, ref):
	if not token.canOperateOnPackage(package):
		return error(403, "API token does not have access to the package")

	if not package.checkPerm(token.owner, Permission.MAKE_RELEASE):
		return error(403, "Permission denied. Missing MAKE_RELEASE permission")

	five_minutes_ago = datetime.datetime.now() - datetime.timedelta(minutes=5)
	count = package.releases.filter(PackageRelease.releaseDate > five_minutes_ago).count()
	if count >= 2:
		return error(429, "Too many requests, please wait before trying again")

	rel = PackageRelease()
	rel.package = package
	rel.title   = title
	rel.url     = ""
	rel.task_id = uuid()
	rel.min_rel = None
	rel.max_rel = None
	db.session.add(rel)
	db.session.commit()

	makeVCSRelease.apply_async((rel.id, ref), task_id=rel.task_id)

	return jsonify({
		"success": True,
		"task": url_for("tasks.check", id=rel.task_id),
		"release": rel.getAsDictionary()
	}) 

def create_release(package):
	if not package.checkPerm(current_user, Permission.MAKE_RELEASE):
		return redirect(package.getDetailsURL())

	# Initial form class from post data and default data
	form = CreatePackageReleaseForm()
	if package.repo is not None:
		form["uploadOpt"].choices = [("vcs", "From Git Commit or Branch"), ("upload", "File Upload")]
		if request.method != "POST":
			form["uploadOpt"].data = "vcs"

	if request.method == "POST" and form.validate():
		if form["uploadOpt"].data == "vcs":
			rel = PackageRelease()
			rel.package = package
			rel.title   = form["title"].data
			rel.url     = ""
			rel.task_id = uuid()
			rel.min_rel = form["min_rel"].data.getActual()
			rel.max_rel = form["max_rel"].data.getActual()
			db.session.add(rel)
			db.session.commit()

			makeVCSRelease.apply_async((rel.id, form["vcsLabel"].data), task_id=rel.task_id)

			msg = "{}: Release {} created".format(package.title, rel.title)
			triggerNotif(package.author, current_user, msg, rel.getEditURL())
			db.session.commit()

			return redirect(url_for("tasks.check", id=rel.task_id, r=rel.getEditURL()))
		else:
			uploadedUrl, uploadedPath = doFileUpload(form.fileUpload.data, "zip", "a zip file")
			if uploadedUrl is not None:
				rel = PackageRelease()
				rel.package = package
				rel.title = form["title"].data
				rel.url = uploadedUrl
				rel.task_id = uuid()
				rel.min_rel = form["min_rel"].data.getActual()
				rel.max_rel = form["max_rel"].data.getActual()
				db.session.add(rel)
				db.session.commit()

				checkZipRelease.apply_async((rel.id, uploadedPath), task_id=rel.task_id)

				msg = "{}: Release {} created".format(package.title, rel.title)
				triggerNotif(package.author, current_user, msg, rel.getEditURL())
				db.session.commit()

				return redirect(url_for("tasks.check", id=rel.task_id, r=rel.getEditURL()))

	return render_template("packages/release_new.html", package=package, form=form) 

def invia_mail_forzato(self, pk_tuple):
    """
    Questo task invia forzatamente la mail.
    Nessuna verifica si fa se il messaggio è stato precedentemente inviato
    oppure sia accodato. (come con l'invio normale nella funzione di sopra)
    """
    from celery import uuid
    from celery.result import AsyncResult
    from .models import Messaggio

    logger = get_task_logger(__name__)
    rescheduled_tasks_id = list()

    messages_to_resend = Messaggio.objects.filter(pk__in=pk_tuple)
    for msg in messages_to_resend:
        pk = msg.pk

        logger.info("[forced] Controllo messaggio id=%d" % pk)

        # Controllo se il messaggio ha un task_id,
        # se presente - dimentico il task per assegnare un nuovo task_id al messaggio
        if msg.task_id is not None:
            task = AsyncResult(msg.task_id)
            task.forget()
            logger.info("[forced] Dimentico task_id %s per il messaggio id=%d" % (msg.task_id, pk))

        # Creiamo un nuovo task ID e riaccodiamo il task.
        msg.task_id = uuid()
        msg.save()

        logger.warning("[forced] Nuovo task per l'invio accodato con id=%s." % msg.task_id)

        is_sent = msg.invia(forced=True)
        if not is_sent:
            logger.error("[forced] Errore temporaneo, nuovo tentativo richiesto.")
            raise self.retry()

        # Messaggio inviato con successo.
        logger.info("[forced] Messaggio %s inviato. Rimuovo task_id e salvo." % msg.pk)

        rescheduled_tasks_id.append(msg.task_id)
        msg.task_id = None

        msg.save()

    return len(rescheduled_tasks_id) 

def rischedula_invii_falliti(self, massimo_messaggi=None):
    """
    Un task celery per riaccodare l'invio di tutti i messaggi che non sono stati inviati,
     e non sono attualmente in coda per l'invio (ad esempio, in seguito allo svuotamento
     della coda di posta).

    :param self:                Task celery.
    :param massimo_messaggi:    Il numero massimo di messaggi da riaccodare per l'invio.
    """
    from .models import Messaggio
    from .tasks import invia_mail

    riaccodati = 0
    non_riaccodati = 0

    # Per ognuno dei messaggi in coda
    for messaggio in Messaggio.in_coda():

        # Assicuriamoci di avere l'ultima copia del messaggio
        messaggio.refresh_from_db()
        logger.debug("Controllo messaggio id=%d" % (messaggio.pk,))

        # Se il messaggio ha un task_id associato, assicuriamoci che il task sia in esecuzione
        task = None
        if messaggio.task_id is not None:
            # Controlla lo stato del task Celery
            # http://docs.celeryproject.org/en/latest/reference/celery.result.html#celery.result.AsyncResult.state
            task = self.app.AsyncResult(messaggio.task_id)
            if task.state in ("QUEUED", "STARTED", "RETRY", "FAILURE", "SUCCESS"):
                # Il task e' ancora in coda. Ignora.
                non_riaccodati += 1
                logger.debug("Task (id=%s) gia' in corso - Ignoro." % (messaggio.task_id,))
                continue

            # Dimentichiamoci del task che e' morto
            assert task.state == "PENDING"
            logger.warning("Task (id=%s) apparentemente morto (%s). Dimentica e riavvia." % (messaggio.task_id,
                                                                                             task.state))
            task.forget()

        # Il task e' morto. Ad esempio, la coda e' stata svuotata.
        # Creiamo un nuovo task ID e riaccodiamo il task.
        messaggio.task_id = uuid()
        messaggio.save()

        logger.warning("Nuovo task per l'invio accodato con id=%s." % (messaggio.task_id,))
        invia_mail.apply_async((messaggio.pk,), task_id=messaggio.task_id)

        riaccodati += 1

    return "%d gia' in invio, %d riaccodati per l'invio" % (non_riaccodati, riaccodati) 

def task_file_create():
    """Endpoint for full analysis task."""
    if 'file' not in request.files:
        # no file
        res = ErrorAPIResponse(2020).to_dict()
        return jsonify(res), 400

    file = request.files['file']

    if file.filename == '':
        # noname file
        res = ErrorAPIResponse(2021).to_dict()
        return jsonify(res), 400

    # get pretty print parameter
    pretty = False
    if 'pretty' in request.form:
        pretty = request.form['pretty']
        if pretty not in ('true', 'false'):
            res = ErrorAPIResponse(2000).to_dict()
            return jsonify(res), 400

    # ger exec time parameter
    exec_time = 20

    if 'exec_time' in request.form:
        try:
            exec_time = int(request.form['exec_time'])
        except ValueError:
            res = ErrorAPIResponse(2022).to_dict()
            return jsonify(res), 400

        if (
            exec_time < dynamic_config['min_exectime']
            or exec_time > dynamic_config['max_exectime']
        ):
            res = ErrorAPIResponse(2022).to_dict()
            return jsonify(res), 400

    task_id = uuid()

    # prepare directory and save file
    os.mkdir(f'{storage_path}/{task_id}')
    file_path = f'{storage_path}/{task_id}/{file.filename}'
    file.save(file_path)

    # run analysis
    args = (file_path,)
    kwargs = {'pretty': pretty, 'exec_time': exec_time}
    tasks.full_analysis.apply_async(args, kwargs, task_id=task_id)

    res = {
        'task_id': task_id
    }
    return jsonify(res)