Python rq.get_current_job() Examples

The following are 28 code examples of rq.get_current_job(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module rq , or try the search function .
Example #1
Source File: tasks.py    From recon-ng with GNU General Public License v3.0 6 votes vote down vote up
def run_module(workspace, module):

    results = {}
    try:
        # instantiate important objects
        job = get_current_job()
        recon = base.Recon(check=False, analytics=False, marketplace=False)
        recon.start(base.Mode.JOB, workspace=workspace)
        tasks = Tasks(recon)
        # update the task's status
        tasks.update_task(job.get_id(), status=job.get_status())
        # execute the task
        module = recon._loaded_modules.get(module)
        module.run()
    except Exception as e:
        results['error'] = {
            'type': str(type(e)),
            'message': str(e),
            'traceback': traceback.format_exc(),
        }
    results['summary'] = module._summary_counts
    # update the task's status and results
    tasks.update_task(job.get_id(), status='finished', result=results)
    return results 
Example #2
Source File: jobs.py    From gigantum-client with MIT License 6 votes vote down vote up
def test_sleep(n):
    """Used only for testing -- example method with argument. """
    logger = LMLogger.get_logger()
    logger.info("Starting test_sleep({}) in pid {}".format(n, os.getpid()))

    try:
        job = get_current_job()
        job.meta['sample'] = 'test_sleep metadata'
        job.meta['pid'] = int(os.getpid())
        job.save_meta()

        time.sleep(n)
        logger.info("Completed test_sleep in pid {}".format(os.getpid()))
        return 0
    except Exception as e:
        logger.error("Error on test_sleep in pid {}: {}".format(os.getpid(), e))
        raise 
Example #3
Source File: tasks.py    From django-test-rq with The Unlicense 6 votes vote down vote up
def long_runnig_task(task):
    job = get_current_job()
    task.job_id = job.get_id()
    
    task.result = 'STARTED'
    
    duration_in_second_persentages = task.duration*1.0 / 100
    for i in range(100):
        import time
        task.progress = i
        task.save()
        print task.progress
        time.sleep(duration_in_second_persentages)
    
    task.result = 'FINISHED'
    task.save()
    return task.result 
Example #4
Source File: tasks.py    From django-test-rq with The Unlicense 6 votes vote down vote up
def scheduled_get_url_words(url):
    """
    This creates a ScheduledTask instance for each group of 
    scheduled task - each time this scheduled task is run
    a new instance of ScheduledTaskInstance will be created
    """
    job = get_current_job()

    task, created = ScheduledTask.objects.get_or_create(
        job_id=job.get_id(),
        name=url
    )
    response = requests.get(url)
    response_len = len(response.text)
    ScheduledTaskInstance.objects.create(
        scheduled_task=task,
        result = response_len,
    )
    return response_len 
Example #5
Source File: __init__.py    From ok with Apache License 2.0 5 votes vote down vote up
def background_job(f):
    @functools.wraps(f)
    def job_handler(*args, **kwargs):
        job = get_current_job()
        job.status = 'running'
        db.session.commit()

        stream = io.StringIO()
        handler = JobLogHandler(stream, job)
        logger = get_job_logger()

        logger.setLevel(logging.INFO)
        logger.addHandler(handler)
        return_value = None

        try:
            return_value = f(*args, **kwargs)
        except Exception:
            job.failed = True
            logger.exception('Job failed')

        job.status = 'finished'
        job.result = _format_result(job, return_value)
        job.log = handler.contents
        stream.close()
        db.session.commit()

    return job_handler 
Example #6
Source File: __init__.py    From ok with Apache License 2.0 5 votes vote down vote up
def get_job_logger():
    return logging.getLogger('{}.job_{}'.format(__name__, get_current_job().id)) 
Example #7
Source File: __init__.py    From ok with Apache License 2.0 5 votes vote down vote up
def get_current_job():
    rq_job = rq.get_current_job(connection=get_connection())
    return Job.query.get(rq_job.id) 
Example #8
Source File: jobs.py    From upribox with GNU General Public License v3.0 5 votes vote down vote up
def job_clear_messages():
    job = get_current_job(connection=django_rq.get_connection())
    job.meta['messages'] = deque()
    job.save_meta()
    job.save() 
Example #9
Source File: jobs.py    From upribox with GNU General Public License v3.0 5 votes vote down vote up
def job_message(message, status="success"):
    job = get_current_job(connection=django_rq.get_connection())
    if not job.meta.get('messages'):
        job.meta['messages'] = deque()
    job.meta['messages'].append({"message": message, "status": status})
    job.save_meta()
    job.save() 
Example #10
Source File: dataset_jobs.py    From gigantum-client with MIT License 5 votes vote down vote up
def hash_dataset_files(logged_in_username: str, dataset_owner: str, dataset_name: str,
                       file_list: List, config_file: str = None) -> None:
    """

    Args:
        logged_in_username: username for the currently logged in user
        dataset_owner: Owner of the labbook if this dataset is linked
        dataset_name: Name of the labbook if this dataset is linked
        file_list: List of files to be hashed
        config_file: Optional config file to use

    Returns:
        None
    """
    logger = LMLogger.get_logger()

    p = os.getpid()
    try:
        logger.info(f"(Job {p}) Starting hash_dataset_files(logged_in_username={logged_in_username},"
                    f"dataset_owner={dataset_owner}, dataset_name={dataset_name}")

        ds = InventoryManager(config_file=config_file).load_dataset(logged_in_username, dataset_owner, dataset_name)
        manifest = Manifest(ds, logged_in_username)

        hash_result, fast_hash_result = manifest.hash_files(file_list)

        job = get_current_job()
        if job:
            job.meta['hash_result'] = ",".join(['None' if v is None else v for v in hash_result])
            job.meta['fast_hash_result'] = ",".join(['None' if v is None else v for v in fast_hash_result])
            job.save_meta()

    except Exception as err:
        logger.error(f"(Job {p}) Error in clean_dataset_file_cache job")
        logger.exception(err)
        raise 
Example #11
Source File: jobs.py    From gigantum-client with MIT License 5 votes vote down vote up
def import_dataset_from_zip(archive_path: str, username: str, owner: str,
                            config_file: Optional[str] = None) -> str:
    """Method to import a dataset from a zip file

    Args:
        archive_path(str): Path to the uploaded zip
        username(str): Username
        owner(str): Owner username
        config_file(str): Optional path to a labmanager config file

    Returns:
        str: directory path of imported labbook
    """

    def update_meta(msg):
        job = get_current_job()
        if not job:
            return
        job.meta['feedback'] = msg
        job.save_meta()

    p = os.getpid()
    logger = LMLogger.get_logger()
    logger.info(f"(Job {p}) Starting import_dataset_from_zip(archive_path={archive_path},"
                f"username={username}, owner={owner}, config_file={config_file})")

    try:
        lb = ZipExporter.import_dataset(archive_path, username, owner,
                                        config_file=config_file,
                                        update_meta=update_meta)
        return lb.root_dir
    except Exception as e:
        logger.exception(f"(Job {p}) Error on import_dataset_from_zip({archive_path}): {e}")
        raise
    finally:
        if os.path.exists(archive_path):
            os.remove(archive_path) 
Example #12
Source File: jobs.py    From gigantum-client with MIT License 5 votes vote down vote up
def import_labboook_from_zip(archive_path: str, username: str, owner: str,
                             config_file: Optional[str] = None) -> str:
    """Method to import a labbook from a zip file

    Args:
        archive_path(str): Path to the uploaded zip
        username(str): Username
        owner(str): Owner username
        config_file(str): Optional path to a labmanager config file

    Returns:
        str: directory path of imported labbook
    """

    def update_meta(msg):
        job = get_current_job()
        if not job:
            return
        job.meta['feedback'] = msg
        job.save_meta()

    p = os.getpid()
    logger = LMLogger.get_logger()
    logger.info(f"(Job {p}) Starting import_labbook_from_zip(archive_path={archive_path},"
                f"username={username}, owner={owner}, config_file={config_file})")

    try:
        lb = ZipExporter.import_labbook(archive_path, username, owner,
                                        config_file=config_file,
                                        update_meta=update_meta)
        return lb.root_dir
    except Exception as e:
        logger.exception(f"(Job {p}) Error on import_labbook_from_zip({archive_path}): {e}")
        raise
    finally:
        if os.path.exists(archive_path):
            os.remove(archive_path) 
Example #13
Source File: jobs.py    From gigantum-client with MIT License 5 votes vote down vote up
def import_labbook_from_remote(remote_url: str, username: str, config_file: str = None) -> str:
    """Return the root directory of the newly imported Project

    Args:
        remote_url: Canonical world-facing URI, like "https://repo.domain/owner/project". This will be converted to the
          actual network location for our repository, like "https://username@repo.domain/owner/project.git/", as
          robustly as we can manage.
        username: username for currently logged in user
        config_file: a copy of the parsed config file

    Returns:
        Path to project root directory
    """
    p = os.getpid()
    logger = LMLogger.get_logger()
    logger.info(f"(Job {p}) Starting import_labbook_from_remote({remote_url}, {username})")

    def update_meta(msg):
        job = get_current_job()
        if not job:
            return
        if 'feedback' not in job.meta:
            job.meta['feedback'] = msg
        else:
            job.meta['feedback'] = job.meta['feedback'] + f'\n{msg}'
        job.save_meta()

    remote = RepoLocation(remote_url, username)
    update_meta(f"Importing Project from {remote.owner_repo!r}...")

    try:
        wf = LabbookWorkflow.import_from_remote(remote, username, config_file)
    except Exception as e:
        update_meta(f"Could not import Project from {remote.remote_location}.")
        logger.exception(f"(Job {p}) Error on import_labbook_from_remote: {e}")
        raise

    update_meta(f"Imported Project {wf.labbook.name}!")
    return wf.labbook.root_dir 
Example #14
Source File: jobs.py    From gigantum-client with MIT License 5 votes vote down vote up
def sync_repository(repository: Repository, username: str, override: MergeOverride,
                    remote: str = "origin", access_token: str = None,
                    pull_only: bool = False, id_token: str = None) -> int:
    p = os.getpid()
    logger = LMLogger.get_logger()
    logger.info(f"(Job {p}) Starting sync_repository({str(repository)})")

    def update_feedback(msg: str, has_failures: Optional[bool] = None, failure_detail: Optional[str] = None,
                        percent_complete: Optional[float] = None):
        """Method to update the job's metadata and provide feedback to the UI"""
        current_job = get_current_job()
        if not current_job:
            return
        if has_failures:
            current_job.meta['has_failures'] = has_failures
        if failure_detail:
            current_job.meta['failure_detail'] = failure_detail
        if percent_complete:
            current_job.meta['percent_complete'] = percent_complete

        current_job.meta['feedback'] = msg
        current_job.save_meta()

    try:
        update_feedback("Sync task in queue")
        with repository.lock():
            if isinstance(repository, LabBook):
                wf = LabbookWorkflow(repository)
            else:
                wf = DatasetWorkflow(repository)  # type: ignore
            cnt = wf.sync(username=username, remote=remote, override=override,
                          feedback_callback=update_feedback, access_token=access_token,
                          id_token=id_token, pull_only=pull_only)
        logger.info(f"(Job {p} Completed sync_repository with cnt={cnt}")
        return cnt
    except MergeConflict as me:
        logger.exception(f"(Job {p}) Merge conflict: {me}")
        raise 
Example #15
Source File: jobs.py    From gigantum-client with MIT License 5 votes vote down vote up
def publish_repository(repository: Repository, username: str, access_token: str,
                       remote: Optional[str] = None, public: bool = False, id_token: str = None) -> None:
    p = os.getpid()
    logger = LMLogger.get_logger()
    logger.info(f"(Job {p}) Starting publish_repository({str(repository)})")

    def update_feedback(msg: str, has_failures: Optional[bool] = None, failure_detail: Optional[str] = None,
                        percent_complete: Optional[float] = None):
        """Method to update the job's metadata and provide feedback to the UI"""
        current_job = get_current_job()
        if not current_job:
            return
        if has_failures:
            current_job.meta['has_failures'] = has_failures
        if failure_detail:
            current_job.meta['failure_detail'] = failure_detail
        if percent_complete:
            current_job.meta['percent_complete'] = percent_complete

        current_job.meta['feedback'] = msg
        current_job.save_meta()

    update_feedback("Publish task in queue")
    with repository.lock():
        if isinstance(repository, LabBook):
            wf = LabbookWorkflow(repository)
        else:
            wf = DatasetWorkflow(repository)  # type: ignore
        wf.publish(username=username, access_token=access_token, remote=remote or "origin",
                   public=public, feedback_callback=update_feedback, id_token=id_token) 
Example #16
Source File: rq_test.py    From busy-beaver with MIT License 5 votes vote down vote up
def add(x, y):
    job = get_current_job()
    task = Task(job_id=job.id, name="Test Task", description="Trying things out")
    db.session.add(task)
    db.session.commit()

    set_task_progress(100)
    return x + y 
Example #17
Source File: rq.py    From busy-beaver with MIT License 5 votes vote down vote up
def set_task_progress(progress):
    job = get_current_job()
    if job:
        job.meta["progress"] = progress
        job.save_meta()

        if progress >= 100:
            task = Task.query.filter_by(job_id=job.get_id()).first()
            if task:
                task.complete = True
                db.session.commit() 
Example #18
Source File: tasks.py    From django-test-rq with The Unlicense 5 votes vote down vote up
def get_url_words(url):
    # This creates a Task instance to save the job instance and job result
    job = get_current_job()

    task = Task.objects.create(
        job_id=job.get_id(),
        name=url
    )
    response = requests.get(url)
    task.result = len(response.text)
    task.save()
    return task.result 
Example #19
Source File: treeitem.py    From zing with GNU General Public License v3.0 5 votes vote down vote up
def update_cache_job(instance):
    """RQ job"""
    job = get_current_job()
    job_wrapper = JobWrapper(job.id, job.connection)
    keys, decrement = job_wrapper.get_job_params()

    # close unusable and obsolete connections before and after the job
    # Note: setting CONN_MAX_AGE parameter can have negative side-effects
    # CONN_MAX_AGE value should be lower than DB wait_timeout
    connection.close_if_unusable_or_obsolete()
    instance._update_cache_job(keys, decrement)
    connection.close_if_unusable_or_obsolete()

    job_wrapper.clear_job_params() 
Example #20
Source File: treeitem.py    From zing with GNU General Public License v3.0 5 votes vote down vote up
def unregister_dirty(self, decrement=1):
        """Unregister current TreeItem as dirty
        (should be called from RQ job procedure after cache is updated)
        """
        r_con = get_connection()
        job = get_current_job()
        if job:
            logger.debug(
                "UNREGISTER %s (-%s) where job_id=%s", self.cache_key, decrement, job.id
            )
        else:
            logger.debug("UNREGISTER %s (-%s)", self.cache_key, decrement)
        r_con.zincrby(KEY_DIRTY_TREEITEMS, 0 - decrement, self.cache_key) 
Example #21
Source File: treeitem.py    From zing with GNU General Public License v3.0 5 votes vote down vote up
def unregister_all_dirty(self, decrement=1):
        """Unregister current TreeItem and all parent paths as dirty
        (should be called from RQ job procedure after cache is updated)
        """
        r_con = get_connection()
        job = get_current_job()
        for p in self.all_pootle_paths():
            if job:
                logger.debug(
                    "UNREGISTER %s (-%s) where job_id=%s", p, decrement, job.id
                )
            else:
                logger.debug("UNREGISTER %s (-%s)", p, decrement)
            r_con.zincrby(KEY_DIRTY_TREEITEMS, 0 - decrement, p) 
Example #22
Source File: tasks.py    From docsbox with MIT License 5 votes vote down vote up
def process_document(path, options, meta):
    current_task = get_current_job()
    with Office(app.config["LIBREOFFICE_PATH"]) as office: # acquire libreoffice lock
        with office.documentLoad(path) as original_document: # open original document
            with TemporaryDirectory() as tmp_dir: # create temp dir where output'll be stored
                for fmt in options["formats"]: # iterate over requested formats
                    current_format = app.config["SUPPORTED_FORMATS"][fmt]
                    output_path = os.path.join(tmp_dir, current_format["path"])
                    original_document.saveAs(output_path, fmt=current_format["fmt"])
                if options.get("thumbnails", None):
                    is_created = False
                    if meta["mimetype"] == "application/pdf":
                        pdf_path = path
                    elif "pdf" in options["formats"]:
                        pdf_path = os.path.join(tmp_dir, "pdf")
                    else:
                        pdf_tmp_file = NamedTemporaryFile()
                        pdf_path = pdf_tmp_file.name
                        original_document.saveAs(pdf_tmp_file.name, fmt="pdf")
                        is_created = True
                    image = Image(filename=pdf_path,
                                  resolution=app.config["THUMBNAILS_DPI"])
                    if is_created:
                        pdf_tmp_file.close()
                    thumbnails = make_thumbnails(image, tmp_dir, options["thumbnails"]["size"])
                result_path, result_url = make_zip_archive(current_task.id, tmp_dir)
        remove_file.schedule(
            datetime.timedelta(seconds=app.config["RESULT_FILE_TTL"]),
            result_path
        )
    return result_url 
Example #23
Source File: jobs.py    From gigantum-client with MIT License 4 votes vote down vote up
def update_unmanaged_dataset_from_remote(logged_in_username: str, access_token: str, id_token: str,
                                         dataset_owner: str, dataset_name: str) -> None:
    """Method to update/populate an unmanaged dataset from its remote automatically

    Args:
        logged_in_username: username for the currently logged in user
        access_token: bearer token
        id_token: identity token
        dataset_owner: Owner of the dataset containing the files to download
        dataset_name: Name of the dataset containing the files to download

    Returns:

    """
    def update_meta(msg):
        job = get_current_job()
        if not job:
            return
        if 'feedback' not in job.meta:
            job.meta['feedback'] = msg
        else:
            job.meta['feedback'] = job.meta['feedback'] + f'\n{msg}'
        job.save_meta()

    logger = LMLogger.get_logger()

    try:
        p = os.getpid()
        logger.info(f"(Job {p}) Starting update_unmanaged_dataset_from_remote(logged_in_username={logged_in_username},"
                    f"dataset_owner={dataset_owner}, dataset_name={dataset_name}")

        im = InventoryManager()
        ds = im.load_dataset(logged_in_username, dataset_owner, dataset_name)

        ds.namespace = dataset_owner
        ds.backend.set_default_configuration(logged_in_username, access_token, id_token)

        if not isinstance(ds.backend, UnmanagedStorageBackend):
            raise ValueError("Can only auto-update unmanaged dataset types")

        if not ds.backend.can_update_from_remote:
            raise ValueError("Storage backend cannot update automatically from remote.")

        ds.backend.update_from_remote(ds, update_meta)

    except Exception as err:
        logger.exception(err)
        raise 
Example #24
Source File: jobs.py    From gigantum-client with MIT License 4 votes vote down vote up
def verify_dataset_contents(logged_in_username: str, access_token: str, id_token: str,
                            dataset_owner: str, dataset_name: str,
                            labbook_owner: Optional[str] = None, labbook_name: Optional[str] = None) -> None:
    """Method to update/populate an unmanaged dataset from it local state

    Args:
        logged_in_username: username for the currently logged in user
        access_token: bearer token
        id_token: identity token
        dataset_owner: Owner of the dataset containing the files to download
        dataset_name: Name of the dataset containing the files to download
        labbook_owner: Owner of the labbook if this dataset is linked
        labbook_name: Name of the labbook if this dataset is linked

    Returns:
        None
    """
    job = get_current_job()

    def update_meta(msg):
        if not job:
            return
        if 'feedback' not in job.meta:
            job.meta['feedback'] = msg
        else:
            job.meta['feedback'] = job.meta['feedback'] + f'\n{msg}'
        job.save_meta()

    logger = LMLogger.get_logger()

    try:
        p = os.getpid()
        logger.info(f"(Job {p}) Starting verify_dataset_contents(logged_in_username={logged_in_username},"
                    f"dataset_owner={dataset_owner}, dataset_name={dataset_name},"
                    f"labbook_owner={labbook_owner}, labbook_name={labbook_name}")

        im = InventoryManager()
        if labbook_owner is not None and labbook_name is not None:
            # This is a linked dataset, load repo from the Project
            lb = im.load_labbook(logged_in_username, labbook_owner, labbook_name)
            dataset_dir = os.path.join(lb.root_dir, '.gigantum', 'datasets', dataset_owner, dataset_name)
            ds = im.load_dataset_from_directory(dataset_dir)
        else:
            # this is a normal dataset. Load repo from working dir
            ds = im.load_dataset(logged_in_username, dataset_owner, dataset_name)

        ds.namespace = dataset_owner
        ds.backend.set_default_configuration(logged_in_username, access_token, id_token)

        result = ds.backend.verify_contents(ds, update_meta)
        job.meta['modified_keys'] = result

    except Exception as err:
        logger.exception(err)
        raise 
Example #25
Source File: jobs.py    From gigantum-client with MIT License 4 votes vote down vote up
def update_unmanaged_dataset_from_local(logged_in_username: str, access_token: str, id_token: str,
                                        dataset_owner: str, dataset_name: str) -> None:
    """Method to update/populate an unmanaged dataset from it local state

    Args:
        logged_in_username: username for the currently logged in user
        access_token: bearer token
        id_token: identity token
        dataset_owner: Owner of the dataset containing the files to download
        dataset_name: Name of the dataset containing the files to download

    Returns:

    """
    def update_meta(msg):
        job = get_current_job()
        if not job:
            return
        if 'feedback' not in job.meta:
            job.meta['feedback'] = msg
        else:
            job.meta['feedback'] = job.meta['feedback'] + f'\n{msg}'
        job.save_meta()

    logger = LMLogger.get_logger()

    try:
        p = os.getpid()
        logger.info(f"(Job {p}) Starting update_unmanaged_dataset_from_local(logged_in_username={logged_in_username},"
                    f"dataset_owner={dataset_owner}, dataset_name={dataset_name}")

        im = InventoryManager()
        ds = im.load_dataset(logged_in_username, dataset_owner, dataset_name)

        ds.namespace = dataset_owner
        ds.backend.set_default_configuration(logged_in_username, access_token, id_token)

        if not isinstance(ds.backend, UnmanagedStorageBackend):
            raise ValueError("Can only auto-update unmanaged dataset types")

        ds.backend.update_from_local(ds, update_meta, verify_contents=True)

    except Exception as err:
        logger.exception(err)
        raise 
Example #26
Source File: jobs.py    From ckanext-xloader with GNU Affero General Public License v3.0 4 votes vote down vote up
def xloader_data_into_datastore(input):
    '''This is the func that is queued. It is a wrapper for
    xloader_data_into_datastore_, and makes sure it finishes by calling
    xloader_hook to update the task_status with the result.

    Errors are stored in task_status and job log and this method returns
    'error' to let RQ know too. Should task_status fails, then we also return
    'error'.
    '''
    # First flag that this task is running, to indicate the job is not
    # stillborn, for when xloader_submit is deciding whether another job would
    # be a duplicate or not
    job_dict = dict(metadata=input['metadata'],
                    status='running')
    callback_xloader_hook(result_url=input['result_url'],
                          api_key=input['api_key'],
                          job_dict=job_dict)

    job_id = get_current_job().id
    errored = False
    try:
        xloader_data_into_datastore_(input, job_dict)
        job_dict['status'] = 'complete'
        db.mark_job_as_completed(job_id, job_dict)
    except JobError as e:
        db.mark_job_as_errored(job_id, str(e))
        job_dict['status'] = 'error'
        job_dict['error'] = str(e)
        log = logging.getLogger(__name__)
        log.error('xloader error: {0}, {1}'.format(e, traceback.format_exc()))
        errored = True
    except Exception as e:
        db.mark_job_as_errored(
            job_id, traceback.format_tb(sys.exc_traceback)[-1] + repr(e))
        job_dict['status'] = 'error'
        job_dict['error'] = str(e)
        log = logging.getLogger(__name__)
        log.error('xloader error: {0}, {1}'.format(e, traceback.format_exc()))
        errored = True
    finally:
        # job_dict is defined in xloader_hook's docstring
        is_saved_ok = callback_xloader_hook(result_url=input['result_url'],
                                            api_key=input['api_key'],
                                            job_dict=job_dict)
        errored = errored or not is_saved_ok
    return 'error' if errored else None 
Example #27
Source File: jobs.py    From grimoirelab-kingarthur with GNU General Public License v3.0 4 votes vote down vote up
def execute_perceval_job(backend, backend_args, qitems, task_id, job_number,
                         category, archive_args=None):
    """Execute a Perceval job on RQ.

    The items fetched during the process will be stored in a
    Redis queue named `queue`.

    Setting the parameter `archive_path`, raw data will be stored
    with the archive manager. The contents from the archive can
    be retrieved setting the parameter `fetch_from_archive` to `True`,
    too. Take into account this behaviour will be only available
    when the backend supports the use of the archive. If archiving
    is not supported, an `AttributeError` exception will be raised.

    :param backend: backend to execute
    :param backend_args: dict of arguments for running the backend
    :param qitems: name of the RQ queue used to store the items
    :param task_id: identifier of the task linked to this job
    :param job_number: human readable identifier for this job
    :param category: category of the items to retrieve
    :param archive_args: archive arguments

    :returns: a `JobResult` instance

    :raises NotFoundError: raised when the backend is not found
    :raises AttributeError: raised when archiving is not supported but
        any of the archive parameters were set
    """
    rq_job = rq.get_current_job()

    job = PercevalJob(rq_job.id, job_number, task_id, backend, category,
                      rq_job.connection, qitems)

    logger.debug("Running job #%s (task: %s) (%s) (cat:%s)",
                 job.job_id, task_id, backend, category)

    if not job.has_archiving() and archive_args:
        raise AttributeError("archive attributes set but archive is not supported")

    try:
        job.run(backend_args, archive_args=archive_args)
    except AttributeError as e:
        raise e
    except Exception as e:
        rq_job = rq.get_current_job()
        rq_job.meta['result'] = job.result
        rq_job.save_meta()
        logger.debug("Error running job %s (%s) - %s",
                     job.job_id, backend, str(e))
        raise e

    result = job.result

    logger.debug("Job #%s (task: %s) completed (%s) - %s/%s items (%s) fetched",
                 result.job_id, task_id, result.backend,
                 str(result.summary.fetched), str(result.summary.skipped),
                 result.category)

    return result 
Example #28
Source File: rqtasks.py    From xcessiv with Apache License 2.0 4 votes vote down vote up
def start_automated_run(path, automated_run_id):
    """Starts automated run. This will automatically create
    base learners until the run finishes or errors out.

    Args:
        path (str): Path to Xcessiv notebook

        automated_run_id (str): Automated Run ID
    """
    with functions.DBContextManager(path) as session:
        automated_run = session.query(models.AutomatedRun).filter_by(id=automated_run_id).first()
        if not automated_run:
            raise exceptions.UserError('Automated run {} '
                                       'does not exist'.format(automated_run_id))
        automated_run.job_id = get_current_job().id
        automated_run.job_status = 'started'

        session.add(automated_run)
        session.commit()

        try:
            if automated_run.category == 'bayes':
                automatedruns.start_naive_bayes(automated_run, session, path)

            elif automated_run.category == 'tpot':
                automatedruns.start_tpot(automated_run, session, path)

            elif automated_run.category == 'greedy_ensemble_search':
                automatedruns.start_greedy_ensemble_search(automated_run, session, path)

            else:
                raise Exception('Something went wrong. Invalid category for automated run')

            automated_run.job_status = 'finished'
            session.add(automated_run)
            session.commit()

        except:
            session.rollback()
            automated_run.job_status = 'errored'
            automated_run.description['error_type'] = repr(sys.exc_info()[0])
            automated_run.description['error_value'] = repr(sys.exc_info()[1])
            automated_run.description['error_traceback'] = \
                traceback.format_exception(*sys.exc_info())
            session.add(automated_run)
            session.commit()
            raise