Python celery.chain() Examples

def import_task():
    """
    Enter all of the daily import tasks into the queue, where they can grind away from there.

    The import is broken up into tasks for a few reasons: it can be paused by stopping the sims queue if necessary;
    works around the celery task time limit.
    """
    if not settings.DO_IMPORTING_HERE:
        return

    tasks = [
        daily_cleanup.si(),
        fix_unknown_emplids.si(),
        get_role_people.si(),
        import_grads.si(),
        get_update_grads_task(),
        import_offerings.si(continue_import=True),
        import_semester_info.si(),
        import_active_grad_gpas.si(),
        #get_import_offerings_task(),
        #import_combined_sections.si(),
        #send_report.si()
    ]

    chain(*tasks).apply_async() 

def get_import_offerings_tasks():
    """
    Get all of the offerings to import, and build tasks (in groups) to do the work.

    Doesn't actually call the jobs: just returns celery tasks to be called.
    """
    #offerings = importer.import_offerings(extra_where="ct.subject='CMPT' and ct.catalog_nbr IN (' 383', ' 470')")
    #offerings = importer.import_offerings()
    offerings = importer.import_offerings(cancel_missing=True)
    offerings = list(offerings)
    offerings.sort()

    offering_groups = _grouper(offerings, 10)
    slug_groups = ([o.slug for o in offerings] for offerings in offering_groups)

    #tasks = [import_offering_group.si(slugs) for slugs in slug_groups]
    #return tasks

    offering_import_chain = chain(*[import_offering_group.si(slugs) for slugs in slug_groups])
    return offering_import_chain 

def send_commands_to_celery(populated_command_tuple,output_base_dir,simulation):

    celery_path = sys.path[0]
    cmd_name, populated_command, vhost, outfile, workspace, task_id,scanned_service_port, scanned_service_name,scanned_service_protocol = populated_command_tuple
    host_dir = output_base_dir + vhost
    host_data_dir = host_dir + "/celerystalkOutput/"

    utils.create_task(cmd_name, populated_command, vhost, outfile, workspace, task_id)
    result = chain(
        # insert a row into the database to mark the task as submitted. a subtask does not get tracked
        # in celery the same way a task does, for instance, you can't find it in flower
        # tasks.cel_create_task.subtask(args=(cmd_name, populated_command, ip, outfile + ".txt", workspace, task_id)),

        # run the command. run_task takes care of marking the task as started and then completed.
        # The si tells run_cmd to ignore the data returned from a previous task
        tasks.run_cmd.si(cmd_name, populated_command, celery_path, task_id).set(task_id=task_id),
    )()  # .apply_async()

    #task_id_list.append(result.task_id)
    host_audit_log = host_dir + "/" + "{0}_executed_commands.txt".format(vhost)
    f = open(host_audit_log, 'a')
    f.write(populated_command + "\n\n")
    f.close() 

def run(self):
        cause_options = self.cause_agent.options or {}
        cause_options.update({k: v for k, v in self.cause_options.items()
                              if not k.startswith('_')})
        cause_options['task_pair_id'] = self.id
        effect_options = self.effect_agent.options or {}
        effect_options.update({k: v for k, v in self.effect_options.items()
                               if not k.startswith('_')})
        effect_options['task_pair_id'] = self.id

        cause = self.cause.s(**cause_options)

        if getattr(self.cause, 'dedup_key', None) is not None:
            effect = dedup_effect_wrapper.s(
                dedup_key=self.cause.dedup_key,
                task_pair_id=self.id,
                effect=self.effect.s(**effect_options),
            )
        else:
            effect = self.effect.s(**effect_options)

        return chain(cause, dmap.s(effect))() 

def _get_index_analyzers():
    """Get list of index analysis tasks to run.

    Returns:
        Celery chain of index analysis tasks as Celery subtask signatures or
        None if index analyzers are disabled in config.
    """
    tasks = []
    index_analyzers = current_app.config.get('AUTO_INDEX_ANALYZERS')

    if not index_analyzers:
        return None

    for analyzer_name, _ in manager.AnalysisManager.get_analyzers(
            index_analyzers):
        tasks.append(run_index_analyzer.s(analyzer_name))

    return chain(tasks) 

def test_b2c_tasks(self, mock_post, mock_get):
        self.assertTrue(chain(send_b2c_request_task.s(100, 254708374149, 1),
                              process_b2c_call_response_task.s(1)).apply_async()) 

def apply_flags_to_web_service_scan(
        self,
        org_uuid=None,
        web_service_uuid=None,
        web_service_scan_uuid=None,
        order_uuid=None,
):
    """
    Apply all of the relevant flags to the data collected during the given web service scan.
    :param org_uuid: The UUID of the organization that flags are being applied for.
    :param web_service_uuid: The UUID of the web service that was scanned.
    :param web_service_scan_uuid: The UUID of the web service scan to update data for.
    :return: None
    """
    logger.info(
        "Now applying flags to web service scan %s."
        % (web_service_scan_uuid,)
    )
    flags = get_all_web_flags_for_organization(db_session=self.db_session, org_uuid=org_uuid)
    if len(flags) == 0:
        logger.info(
            "No web flags found for organization %s."
            % (org_uuid,)
        )
        return
    task_sigs = []
    for flag in flags:
        flag_type = "default" if isinstance(flag, DefaultFlag) else "organization"
        task_sigs.append(apply_flag_to_web_service_scan.si(
            org_uuid=org_uuid,
            web_service_uuid=web_service_uuid,
            web_service_scan_uuid=web_service_scan_uuid,
            flag_uuid=flag.uuid,
            flag_type=flag_type,
            order_uuid=order_uuid,
        ))
    canvas_sig = chain(task_sigs)
    self.finish_after(signature=canvas_sig)


#USED 

def discover_virtual_hosts_for_web_service(
        self,
        org_uuid=None,
        network_service_scan_uuid=None,
        network_service_uuid=None,
        use_ssl=None,
        order_uuid=None,
):
    """
    Discover all of the virtual hosts for the given web service.
    :param org_uuid: The organization to discover virtual hosts on behalf of.
    :param network_service_scan_uuid: The UUID of the network service scan that this virtual host discovery is
    a part of.
    :param network_service_uuid: The UUID of the network service where the web service resides.
    :param use_ssl: Whether or not to use SSL to interact with the remote web service.
    :return: None
    """
    logger.info(
        "Now discovering virtual hosts for network service %s. Organization is %s, scan is %s."
        % (network_service_uuid, org_uuid, network_service_scan_uuid)
    )
    task_sigs = []
    task_kwargs = {
        "org_uuid": org_uuid,
        "network_service_uuid": network_service_uuid,
        "network_service_scan_uuid": network_service_scan_uuid,
        "use_ssl": use_ssl,
        "order_uuid": order_uuid,
    }
    task_sigs.append(fingerprint_virtual_hosts.si(**task_kwargs))
    task_sigs.append(assess_virtual_host_fingerprints.si(**task_kwargs))
    logger.info(
        "Now kicking off virtual host fingerprinting and assessment tasks for scan %s, organization %s."
        % (network_service_scan_uuid, org_uuid)
    )
    canvas_sig = chain(task_sigs)
    self.finish_after(signature=canvas_sig)


#USED 

def redo_ssl_support_inspection_for_network_service_scan(self, network_service_scan_uuid=None):
    """
    Perform SSL support inspection for the given network service scan again.
    :param network_service_scan_uuid: The UUID of the network service scan to perform SSL support inspection
    for.
    :return: None
    """
    from .base import update_network_service_scan_elasticsearch
    logger.info(
        "Now redo'ing SSL support inspection for network service scan %s."
        % (network_service_scan_uuid,)
    )
    org_uuid, network_service_uuid = get_related_uuids_from_network_service_scan(
        network_service_scan_uuid=network_service_scan_uuid,
        db_session=self.db_session,
    )
    delete_ssl_inspection_documents_for_network_service_scan(
        org_uuid=org_uuid,
        network_service_scan_uuid=network_service_scan_uuid,
    )
    task_sigs = []
    task_sigs.append(inspect_tcp_service_for_ssl_support.si(
        org_uuid=org_uuid,
        network_service_uuid=network_service_uuid,
        network_service_scan_uuid=network_service_scan_uuid,
    ))
    task_sigs.append(update_network_service_scan_elasticsearch.si(
        network_service_scan_uuid=network_service_scan_uuid,
        org_uuid=org_uuid,
        network_service_uuid=network_service_uuid,
    ))
    canvas_sig = chain(task_sigs)
    self.finish_after(signature=canvas_sig)


#USED 

def inspect_service_application(
        self,
        org_uuid=None,
        network_service_scan_uuid=None,
        network_service_uuid=None,
        order_uuid=None,
):
    """
    Inspect the applications residing on the remote service.
    :param org_uuid: The UUID of the organization to inspect the service on behalf of.
    :param network_service_scan_uuid: The UUID of the NetworkScan that invoked this task.
    :param network_service_uuid: The UUID of the NetworkService to inspect.
    :return: None
    """
    logger.info(
        "Inspecting service %s for organization %s. Network scan was %s."
        % (network_service_uuid, org_uuid, network_service_scan_uuid)
    )
    task_signatures = []
    protocol = self.network_service.protocol.lower()
    if protocol == "tcp":
        task_signatures.append(inspect_tcp_service_application.si(
            org_uuid=org_uuid,
            network_service_scan_uuid=network_service_scan_uuid,
            network_service_uuid=network_service_uuid,
            order_uuid=order_uuid,
        ))
    else:
        raise UnsupportedProtocolError("No support for service inspection with protocol %s." % (protocol,))
    canvas_sig = chain(task_signatures)
    canvas_sig.apply_async()


#USED 

def zmap_scan_order(self, order_uuid=None):
    """
    Perform Zmap scans for all necessary ports for the given order.
    :param order_uuid: The UUID of the order to scan.
    :return: None
    """
    port_tuples = get_ports_to_scan_for_scan_config(
        config_uuid=self.scan_config.uuid,
        db_session=self.db_session,
    )
    logger.info(
        "Now scanning order %s for %s total ports."
        % (order_uuid, len(port_tuples))
    )
    task_signatures = []
    scan_signatures = []
    network_scan = create_network_scan_for_organization(
        db_session=self.db_session,
        org_uuid=self.org_uuid,
    )
    self.commit_session()
    for port, protocol in port_tuples:
        scan_signatures.append(zmap_scan_order_for_port.si(
            port=port,
            protocol=protocol,
            order_uuid=order_uuid,
            network_scan_uuid=network_scan.uuid,
        ))
    task_signatures.append(group(scan_signatures))
    task_signatures.append(update_zmap_scan_completed.si(
        scan_uuid=network_scan.uuid,
        org_uuid=self.org_uuid,
        order_uuid=order_uuid,
    ))
    logger.info("Kicking off Zmap subtasks now.")
    canvas_sig = chain(task_signatures)
    canvas_sig.apply_async()


#USED 

def liberate_collect_emails(results, mail_path, options):
    """ Send off data mining tasks """
    msg_tasks = []
    results = results or []
    for result in results:
        inbox = [(mail_path, result['folder'], email_id) for email_id in result['ids']]
        msg_tasks.extend(inbox)

    task_len = len(msg_tasks)

    if task_len > 0:
        msg_tasks = liberate_message.chunks(msg_tasks, 100).group()
        task_group_skew(msg_tasks, step=10)
        msg_tasks = chain(
                        msg_tasks,
                        liberate_convert_box.s(mail_path, options),
                        liberate_fetch_info.s(options),
                        liberate_tarball.s(options),
                        liberation_finish.s(options)
                        )
    else:
        options["noEmails"] = True
        data = {"results": []}
        msg_tasks = chain(
                        liberate_convert_box.s(data, mail_path, options),
                        liberate_fetch_info.s(options),
                        liberate_tarball.s(options),
                        liberation_finish.s(options)
                        )

    async_result = msg_tasks.apply_async()

    lib_status = get_user_model().objects.get(id=options["user"]).liberation
    lib_status.async_result = async_result.id
    lib_status.save() 

def barrier(self, data, merging=False, sequential=False, replace=None, root_docs=None):
    replacement = []
    if isinstance(data[0], basestring):
        data = [data]
    # XXX: ordering might be incorrect because of suffixes. fix by replacing
    # tuples with class
    data = sorted(data, key=lambda x: x[1])
    # merge output from same source documents
    if merging == 'doc':
        for docs, task in izip(cycle(_group_by_prefix(data, root_docs)), replace):
            if sequential:
                task[0]['args'] = [docs]
                replacement.append(chain(signature(t) for t in task))
            else:
                task['args'] = [docs]
                replacement.append(signature(task))
    # merge everything
    elif merging:
        for task in replace:
            if sequential:
                task[0]['args'] = [data]
                replacement.append(chain(signature(t) for t in task))
            else:
                task['args'] = [data]
                replacement.append(signature(task))
    else:
        for ret_val, task in izip(cycle(data), replace):
            if sequential:
                task[0]['args'] = [ret_val]
                replacement.append(chain(signature(t) for t in task))
            else:
                task['args'] = [ret_val]
                replacement.append(signature(task))
    raise self.replace(group(replacement)) 

def loads(payload):
    if payload.get('type') != 'normal':
        raise Exception('celery task loader only support normal mode')
    tasks = payload.get('tasks', [])
    cts = []
    for task in tasks:
        ops = [load(id, task.get('args'), task.get('on_error')) if i == 0 else load(id, None, task.get('on_error')) for
               i, id in enumerate(task['ids'])]
        cts.append(chain(ops))
    callback = payload.get('callback')
    if callback:
        return chord(header=group(cts), body=func.load(callback).s())
    return group(cts) 

def test_online_tasks(self, mock_post, mock_get):
        self.assertTrue(
            chain(call_online_checkout_task.s(254708374149, 100, '', ''),
                  handle_online_checkout_response_task.s(1)).apply_async()) 

def full_pipeline(dataset_id, project_id):
    '''
    Get properties and then get viz specs
    '''
    pipeline = chain([
        ingestion_pipeline(dataset_id, project_id),
        viz_spec_pipeline(dataset_id, project_id, [])
    ])
    return pipeline 

def nmap_scan_subdomain_host(vhost,workspace,simulation,output_base_dir,config_file=None):
    celery_path = sys.path[0]
    config_nmap_options = config_parser.extract_bb_nmap_options(config_file=config_file)
    config = ConfigParser(allow_no_value=True)
    config.read(['config.ini'])

    vhost_explicitly_out_of_scope = lib.db.is_vhost_explicitly_out_of_scope(vhost, workspace)
    output_host_dir = os.path.normpath(os.path.join(output_base_dir, vhost))
    try:
        os.stat(output_host_dir)
    except:
        os.makedirs(output_host_dir)

    output_file = os.path.normpath(os.path.join(output_host_dir, vhost + "_nmap_tcp_scan"))
    if not vhost_explicitly_out_of_scope:
        #print(config_nmap_options)
        cmd_name = "nmap_tcp_scan"
        try:
            if not simulation:
                populated_command = "nmap " + vhost + config_nmap_options + " -oA " + output_file
            else:
                populated_command = "#nmap " + vhost + config_nmap_options + " -oA " + output_file
        except TypeError:
            print("[!] Error: In the config file, there needs to be one, and only one, enabled tcp_scan command in the nmap_commands section.")
            print("[!]        This determines what ports to scan.")
            exit()
        task_id = uuid()
        utils.create_task(cmd_name, populated_command, vhost, output_file, workspace, task_id)
        result = chain(
        tasks.run_cmd.si(cmd_name, populated_command, celery_path, task_id,output_file=output_file,process_nmap=True).set(task_id=task_id),
        )() 

def populate_dedup_events(self):
        if getattr(self.cause, 'dedup_key', None) is not None:
            cause_options = self.cause_agent.options or {}
            cause_options.update({k: v for k, v in self.cause_options.items()
                                if not k.startswith('_')})
            cause_options['task_pair_id'] = self.id

            cause = self.cause.s(**cause_options)
            effect = create_dedup_event.s(task_pair_id=self.id,
                                        dedup_key=self.cause.dedup_key)
            return chain(cause, dmap.s(effect))() 

def create_dns_recon_tasks(domains,simulation,workspace,output_base_dir,out_of_scope_hosts=None,config_file=None):
    workspace_mode = lib.db.get_workspace_mode(workspace)[0][0]
    task_id_list = []
    total_tasks_num = 0
    config, supported_services = config_parser.read_config_ini(config_file)
    celery_path = sys.path[0]
    for domain in domains.split(","):
        for section in config.sections():
            if section == "domain-recon":
                for (cmd_name, cmd) in config.items(section):
                    outfile = output_base_dir + domain + "_" + cmd_name
                    populated_command = cmd.replace("[DOMAIN]", domain).replace("[OUTPUT]", outfile)
                    populated_command = replace_user_config_options(config_file, populated_command)

                    if simulation:
                        populated_command = "#" + populated_command
                    #print(populated_command)

                    # Grab a UUID from celery.utils so that i can assign it to my task at init, which is amazing because
                    # that allows me to pass it to all of the tasks in the chain.
                    task_id = uuid()
                    utils.create_task(cmd_name, populated_command, domain, outfile + ".txt", workspace, task_id)
                    process_domain_tuple = (cmd_name, populated_command, output_base_dir, workspace, domain, simulation, celery_path, workspace_mode)
                    result = chain(
                        # insert a row into the database to mark the task as submitted. a subtask does not get tracked
                        # in celery the same way a task does, for instance, you can't find it in flower
                        #tasks.cel_create_task.subtask(args=(cmd_name, populated_command, domain, "", workspace, task_id)),

                        # run the command. run_task takes care of marking the task as started and then completed.
                        # The si tells run_cmd to ignore the data returned from a previous task
                        tasks.run_cmd.si(cmd_name, populated_command,celery_path,task_id,process_domain_tuple=process_domain_tuple).set(task_id=task_id),
                    )()  # .apply_async()

    total_tasks_num = total_tasks_num + len(task_id_list)
    print("\n\n[+] Summary:\tSubmitted {0} tasks to the [{1}] workspace.".format(total_tasks_num, workspace))
    print("[+]\t\tThere might be additional tasks added to the queue during post processing\n[+]")
    print("[+]\t\tTo keep an eye on things, run one of these commands: \n[+]")
    print("[+]\t\tcelerystalk query [watch]")
    print("[+]\t\tcelerystalk query brief [watch]")
    print("[+]\t\tcelerystalk query summary [watch]\n") 

def test_chain(app):
    with running_worker(app):
        with app.app_context():
            eq_(chain(test_task.s(1, 2), test_task.s(3)).delay().get(
                interval=0.01), 6) 

def aquatone_host(urls_to_screenshot,vhost,workspace,simulation,scan_output_base_file_dir,celery_path,config_file=None):
    print("in aquatone host")
    celery_path = lib.db.get_current_install_path()[0][0]
    config, supported_services = config_parser.read_config_ini(config_file)
    for (cmd_name, cmd) in config.items("screenshots"):
        #print(cmd_name, cmd)
        try:
            if cmd_name == "aquatone":
                outfile = scan_output_base_file_dir + "_" + cmd_name
                filename = "/tmp/" + workspace + "_paths_" + vhost + ".txt"
                populated_command = cmd.replace("[FILE]", filename).replace("[OUTPUT]", outfile)
                populated_command = replace_user_config_options(config_file,populated_command)

                paths = lib.db.get_all_paths_for_host_path_only(vhost,workspace)
                print(str(paths))


                with open(filename, 'w') as paths_tmp_file:
                    #paths_tmp_file.write(str(paths))
                    for line in paths:
                         #print(str(line))
                         paths_tmp_file.write(str(line[0]) + "\n")

                populated_command = cmd.replace("[FILE]", filename).replace("[OUTPUT]", outfile)
                populated_command = replace_user_config_options(config_file,populated_command)

                #print(populated_command)
        except Exception, e:
            print(e)
            print("[!] Error: In the config file, there needs to be one (and only one) enabled aquatone command.")
            exit()


        task_id = uuid()
        utils.create_task(cmd_name, populated_command, vhost, outfile + "/aquatone_report.html", workspace, task_id)
        result = chain(
            tasks.run_cmd.si(cmd_name, populated_command, celery_path, task_id).set(task_id=task_id),
        )() 

def get_update_grads_task():
    """
    Get grad students to import, and build tasks (in groups) to do the work.

    Doesn't actually call the jobs: just returns a celery task to be called.
    """
    active = GradStudent.objects.filter(current_status__in=STATUS_ACTIVE).select_related('person')
    applicants = GradStudent.objects.filter(current_status__in=STATUS_APPLICANT,
                 updated_at__gt=datetime.datetime.now()-datetime.timedelta(days=7)).select_related('person')
    grads = itertools.chain(active, applicants)
    emplids = set(gs.person.emplid for gs in grads)
    emplid_groups = _grouper(emplids, 20)

    grad_import_chain = chain(*[import_grad_group.si(list(emplids)) for emplids in emplid_groups])
    return grad_import_chain 

def test_group_in_chain_json(app):
    """ This test protects against an issue with nested chains and groups when encoding with json.
        the error appears as 'EncodeError: keys must be a string' as described
        in https://github.com/celery/celery/issues/2033.
        Fixed by installing simplejson.
    """
    with running_worker(app):
        with app.app_context():
            task_group = group(task_json.s(i) for i in xrange(10))
            task_chain = chain(task_json.s(1, 2), task_json.s(4), task_group)
            eq_(task_chain.delay().get(interval=0.01),
                [7, 8, 9, 10, 11, 12, 13, 14, 15, 16]) 

def build_index_pipeline(
        file_path='', events='', timeline_name='', index_name='',
        file_extension='', sketch_id=None, only_index=False):
    """Build a pipeline for index and analysis.

    Args:
        file_path: The full path to a file to upload, either a file_path or
            or events need to be defined.
        events: String with the event data, either file_path or events
            needs to be defined.
        timeline_name: Name of the timeline to create.
        index_name: Name of the index to index to.
        file_extension: The file extension of the file.
        sketch_id: The ID of the sketch to analyze.
        only_index: If set to true then only indexing tasks are run, not
            analyzers. This is to be used when uploading data in chunks,
            we don't want to run the analyzers until all chunks have been
            uploaded.

    Returns:
        Celery chain with indexing task (or single indexing task) and analyzer
        task group.
    """
    if not (file_path or events):
        raise RuntimeError(
            'Unable to upload data, missing either a file or events.')
    index_task_class = _get_index_task_class(file_extension)
    index_analyzer_chain = _get_index_analyzers()
    sketch_analyzer_chain = None
    searchindex = SearchIndex.query.filter_by(index_name=index_name).first()

    index_task = index_task_class.s(
        file_path, events, timeline_name, index_name, file_extension)

    if only_index:
        return index_task

    if sketch_id:
        sketch_analyzer_chain, _ = build_sketch_analysis_pipeline(
            sketch_id, searchindex.id, user_id=None)

    # If there are no analyzers just run the indexer.
    if not index_analyzer_chain and not sketch_analyzer_chain:
        return index_task

    if sketch_analyzer_chain:
        if not index_analyzer_chain:
            return chain(
                index_task, run_sketch_init.s(), sketch_analyzer_chain)
        return chain(
            index_task, index_analyzer_chain, run_sketch_init.s(),
            sketch_analyzer_chain)

    if current_app.config.get('ENABLE_EMAIL_NOTIFICATIONS'):
        return chain(
            index_task,
            index_analyzer_chain,
            run_email_result_task.s()
        )

    return chain(index_task, index_analyzer_chain) 

def liberate(user_id, options):
    """ Get set for liberation, expects User object """
    options['user'] = user_id
    user = get_user_model().objects.get(id=user_id)
    lib_status = user.liberation

    tar_type = TAR_TYPES[options.get('compression_type', '0')]

    rstr = get_random_string(7, string.ascii_letters)
    username = user.username + rstr
    username = username.encode("utf-8")
    basename = "%s_%s_%s_%s" % (time.time(), os.getpid(), rstr, hashlib.sha256(username).hexdigest()[:50])
    path = os.path.join(settings.SENDFILE_ROOT, basename)
    tarname = "%s.%s" % (basename, tar_type["ext"])

    # Is this safe enough?
    try:
        os.mkdir(path, 0o700)
    except (IOError, OSError) as error:
        log.info("Couldn't create dir at %s", path)
        raise liberate.retry(exc=error)

    try:
        lib_status.path = tarname
        lib_status.save()
    except IntegrityError:
        os.rmdir(path)
        raise

    options["path"] = path
    options["tarname"] = tarname

    mail_path = os.path.join(path, 'emails')
    # make maildir
    mailbox.Maildir(mail_path, factory=None)

    inbox_tasks = [liberate_inbox.s(mail_path, inbox.id) for inbox in
                   Inbox.objects.filter(user=user, deleted=False).only('id').iterator()]
    if len(inbox_tasks) > 0:
        tasks = chord(
                    inbox_tasks,
                    liberate_collect_emails.s(mail_path, options)
                    )
    else:
        options["noEmails"] = True
        data = {"results": []}
        tasks = chain(
                    liberate_fetch_info.s(data, options),
                    liberate_tarball.s(options),
                    liberation_finish.s(options)
                )

    async_result = tasks.apply_async()

    lib_status.async_result = async_result.id
    lib_status.save() 

def inspect_https_service(
        self,
        org_uuid=None,
        network_service_scan_uuid=None,
        network_service_uuid=None,
        order_uuid=None,
):
    """
    Inspect the HTTPS service running on the given network service on behalf of the given
    organization and network service scan.
    :param org_uuid: The UUID of the organization to inspect the HTTPS service on behalf of.
    :param network_service_scan_uuid: The UUID of the network service scan.
    :param network_service_uuid: The UUID of the network service.
    :return: None
    """
    logger.info(
        "Now inspecting HTTPS service residing on network service %s. Organization is %s."
        % (network_service_uuid, org_uuid)
    )
    scan_config = self.scan_config
    if scan_config.web_app_enum_vhosts:
        task_sigs = []
        task_kwargs = {
            "org_uuid": org_uuid,
            "network_service_scan_uuid": network_service_scan_uuid,
            "network_service_uuid": network_service_uuid,
            "use_ssl": True,
            "order_uuid": order_uuid,
        }
        task_sigs.append(discover_virtual_hosts_for_web_service.si(**task_kwargs))
        task_sigs.append(inspect_virtual_hosts_for_network_service.si(**task_kwargs))
        logger.info(
            "Now kicking off %s tasks to inspect HTTPS service at %s. Organization is %s."
            % (len(task_sigs), network_service_uuid, org_uuid)
        )
        canvas_sig = chain(task_sigs)
        self.finish_after(signature=canvas_sig)
    else:
        populate_and_scan_web_services_from_network_service_scan.si(
            org_uuid=org_uuid,
            network_service_scan_uuid=network_service_scan_uuid,
            network_service_uuid=network_service_uuid,
            order_uuid=order_uuid,
            use_ssl=True,
        ).apply_async()


#USED 

def inspect_http_service(
        self,
        org_uuid=None,
        network_service_scan_uuid=None,
        network_service_uuid=None,
        order_uuid=None,
):
    """
    Inspect the HTTP service running on the given network service on behalf of the given
    organization and network service scan.
    :param org_uuid: The UUID of the organization to inspect the HTTP service on behalf of.
    :param network_service_scan_uuid: The UUID of the network service scan.
    :param network_service_uuid: The UUID of the network service.
    :return: None
    """
    logger.info(
        "Now inspecting HTTP service residing on network service %s. Organization is %s."
        % (network_service_uuid, org_uuid)
    )
    scan_config = self.scan_config
    if scan_config.web_app_enum_vhosts:
        task_sigs = []
        task_kwargs = {
            "org_uuid": org_uuid,
            "network_service_scan_uuid": network_service_scan_uuid,
            "network_service_uuid": network_service_uuid,
            "use_ssl": False,
            "order_uuid": order_uuid,
        }
        task_sigs.append(discover_virtual_hosts_for_web_service.si(**task_kwargs))
        task_sigs.append(inspect_virtual_hosts_for_network_service.si(**task_kwargs))
        logger.info(
            "Now kicking off %s tasks to inspect HTTP service at %s. Organization is %s."
            % (len(task_sigs), network_service_uuid, org_uuid)
        )
        canvas_sig = chain(task_sigs)
        self.finish_after(signature=canvas_sig)
    else:
        populate_and_scan_web_services_from_network_service_scan.si(
            org_uuid=org_uuid,
            network_service_scan_uuid=network_service_scan_uuid,
            network_service_uuid=network_service_uuid,
            use_ssl=False,
            order_uuid=order_uuid,
        ).apply_async()


#USED 

def inspect_tcp_service_for_ssl_support(
        self,
        org_uuid=None,
        network_service_uuid=None,
        network_service_scan_uuid=None,
        order_uuid=None,
):
    """
    Collect all possible information about SSL support found on the referenced network service.
    :param org_uuid: The UUID of the organization to collect information on behalf of.
    :param network_service_uuid: The UUID of the network service to check for SSL support.
    :param network_service_scan_uuid: The UUID of the network service scan that this SSL support check is associated
    with.
    :return: None
    """
    ip_address = self.network_service.ip_address.address
    port = self.network_service.port
    logger.info(
        "Now inspecting TCP service at %s:%s for SSL data for organization %s. Scan is %s."
        % (ip_address, port, org_uuid, network_service_scan_uuid)
    )
    initial_check = self.inspector.check_ssl_support()
    if not initial_check:
        logger.info(
            "Service at %s:%s does not support any version of SSL."
            % (ip_address, port)
        )
        return
    logger.info(
        "Service at %s:%s supports SSL. Now kicking off subtasks to check for various version support."
        % (ip_address, port)
    )
    task_sigs = []
    task_kwargs = {
        "org_uuid": org_uuid,
        "network_service_uuid": network_service_uuid,
        "network_service_scan_uuid": network_service_scan_uuid,
        "order_uuid": order_uuid,
    }
    collection_sigs = []
    scan_config = self.scan_config
    if scan_config.ssl_enumerate_vulns:
        collection_sigs.append(enumerate_vulnerabilities_for_ssl_service.si(**task_kwargs))
    if scan_config.ssl_enumerate_cipher_suites:
        collection_sigs.append(enumerate_cipher_suites_for_ssl_service.si(**task_kwargs))
    if scan_config.ssl_retrieve_cert:
        collection_sigs.append(retrieve_ssl_certificate_for_tcp_service.si(**task_kwargs))
    task_sigs.append(group(collection_sigs))
    task_sigs.append(create_ssl_support_report_for_network_service_scan.si(**task_kwargs))
    task_sigs.append(apply_flags_to_ssl_support_scan.si(**task_kwargs))
    if config.pubsub_enabled:
        task_sigs.append(publish_report_for_ssl_support_scan.si(**task_kwargs))
    canvas_sig = chain(task_sigs)
    logger.info(
        "Now kicking off %s tasks to inspect SSL support for network service %s."
        % (len(collection_sigs) + 1, network_service_uuid)
    )
    self.finish_after(signature=canvas_sig)


#USED 

def fingerprint_tcp_service(
        self,
        org_uuid=None,
        network_service_uuid=None,
        network_service_scan_uuid=None,
        order_uuid=None,
):
    """
    Perform fingerprinting of the given TCP network service.
    :param org_uuid: The UUID of the organization to check the service on behalf of.
    :param network_service_uuid: The UUID of the network service to fingerprint.
    if the network service is found to be alive.
    :param network_service_scan_uuid: The UUID of the network service scan that this service fingerprinting is associated
    with.
    :return: None
    """
    ip_address, port, protocol = self.get_endpoint_information()
    logger.debug(
        "Now performing TCP network service inspection for %s:%s for organization %s. Scan is %s."
        % (ip_address, port, org_uuid, network_service_scan_uuid)
    )
    task_sigs = []
    fingerprinting_sigs = []
    for fingerprinting_task in get_tcp_service_fingerprinting_tasks():
        fingerprinting_sigs.append(fingerprinting_task.si(
            org_uuid=org_uuid,
            ip_address=ip_address,
            port=port,
            network_service_scan_uuid=network_service_scan_uuid,
            network_service_uuid=network_service_uuid,
            order_uuid=order_uuid,
        ))
    supported_ssl_version = get_supported_ssl_version_for_service(
        org_uuid=org_uuid,
        scan_uuid=network_service_scan_uuid,
    )
    if supported_ssl_version is not None:
        for fingerprinting_task in get_tcp_ssl_service_fingerprinting_tasks():
            fingerprinting_sigs.append(fingerprinting_task.si(
                org_uuid=org_uuid,
                ip_address=ip_address,
                port=port,
                network_service_scan_uuid=network_service_scan_uuid,
                ssl_version=supported_ssl_version,
                network_service_uuid=network_service_uuid,
                order_uuid=order_uuid,
            ))
    task_sigs.append(group(fingerprinting_sigs))
    canvas_sig = chain(task_sigs)
    logger.debug(
        "Now kicking off a total of %s tasks to fingerprint TCP service at %s:%s for organization %s."
        % (len(fingerprinting_sigs), ip_address, port, org_uuid)
    )
    self.finish_after(signature=canvas_sig) 

def enumerate_subdomains_for_domain(
        self,
        org_uuid=None,
        domain_uuid=None,
        domain_name=None,
        domain_scan_uuid=None,
        order_uuid=None,
):
    """
    Enumerate subdomains for the given domain name and associate the results with the given domain UUID.
    :param org_uuid: The UUID of the organization to perform the task for.
    :param domain_uuid: The UUID of the parent domain that this subdomain scan is invoked on behalf of.
    :param domain_name: The domain name to enumerate subdomains for.
    :param order_uuid: The UUID of the order that this enumeration is associated with.
    :return: None
    """
    logger.info(
        "Now enumerating subdomains for domain name %s (parent domain %s)."
        % (domain_name, domain_uuid)
    )
    try:
        parent_domain = get_parent_domain_for_subdomain_discovery(domain_name)
    except UnsupportedTldException:
        logger.warning(
            "The domain %s contains a TLD that we do not support."
            % (domain_name,)
        )
        return
    task_sigs = []
    discovery_sigs = []
    task_kwargs = {
        "org_uuid": org_uuid,
        "domain_uuid": domain_uuid,
        "domain_scan_uuid": domain_scan_uuid,
        "parent_domain": parent_domain,
        "order_uuid": order_uuid,
    }
    discovery_sigs.append(enumerate_subdomains_by_dnsdb.si(**task_kwargs))
    task_sigs.append(group(discovery_sigs))
    task_sigs.append(create_and_inspect_domains_from_subdomain_enumeration.si(**task_kwargs))
    canvas_sig = chain(task_sigs)
    self.finish_after(signature=canvas_sig)


#USED