Python celery.exceptions.SoftTimeLimitExceeded() Examples

The following are 14 code examples of celery.exceptions.SoftTimeLimitExceeded(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module celery.exceptions , or try the search function

Example #1

Source File: execute_command.py From scoringengine with MIT License

6 votes

def execute_command(job):
    output = ""
    # Disable duplicate celery log messages
    if logger.propagate:
        logger.propagate = False
    logger.info("Running cmd for " + str(job))
    try:
        cmd_result = subprocess.run(
            job['command'],
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT
        )
        output = cmd_result.stdout.decode("utf-8")
        job['errored_out'] = False
    except SoftTimeLimitExceeded:
        job['errored_out'] = True
    job['output'] = output
    return job

Example #2

Source File: comment.py From weibospider with MIT License

6 votes

def crawl_comment_by_page(mid, page_num):
    try:
        cur_url = BASE_URL.format(mid, page_num)
        html = get_page(cur_url, auth_level=1, is_ajax=True)
        comment_datas = comment.get_comment_list(html, mid)
    except SoftTimeLimitExceeded:
        crawler.error(
            "comment SoftTimeLimitExceeded    mid={mid} page_num={page_num}".
            format(mid=mid, page_num=page_num))
        app.send_task(
            'tasks.comment.crawl_comment_by_page',
            args=(mid, page_num),
            queue='comment_page_crawler',
            routing_key='comment_page_info')
    CommentOper.add_all(comment_datas)
    if page_num == 1:
        WbDataOper.set_weibo_comment_crawled(mid)
    return html, comment_datas

Example #3

Source File: retriever.py From http-observatory with Mozilla Public License 2.0

6 votes

def __get(session, relative_path='/', headers=None, cookies=None):
    if not headers:
        headers = {}

    if not cookies:
        cookies = {}

    try:
        # TODO: limit the maximum size of the response, to keep malicious site operators from killing us
        # TODO: Perhaps we can naively do it for now by simply setting a timeout?
        # TODO: catch TLS errors instead of just setting it to None?
        return session.get(session.url.scheme + '://' + session.url.netloc + relative_path,
                           headers=headers,
                           cookies=cookies,
                           timeout=TIMEOUT)
    # Let celery exceptions percolate upward
    except (SoftTimeLimitExceeded, TimeLimitExceeded):
        raise
    except (KeyboardInterrupt, SystemExit):
        raise
    except:
        return None

Example #4

Source File: scan.py From seecode-scanner with GNU General Public License v3.0

6 votes

def start(**kwargs):
    """
    添加扫描任务
    :param kwargs: 任务配置
    :return:
    """
    pro = None
    try:
        init()
        result_file = '{0}.json'.format(kwargs.get("project_name"))
        kwargs['result_file'] = result_file
        pro = start_scan(**kwargs)
    except SoftTimeLimitExceeded as ex:
        logger.critical(ex)
        if pro:
            pro.update_scan_failed_status(ex)
    except Exception as ex:
        import traceback;traceback.print_exc()
        logger.critical(ex)
    except KeyboardInterrupt:
        logger.error("user aborted")
    except SystemExit:
        raise

    return False

Example #5

Source File: test_execute_command.py From scoringengine with MIT License

5 votes

def test_timed_out(self):
        import subprocess
        subprocess.run = mock.Mock(side_effect=SoftTimeLimitExceeded)

        job = Job(environment_id="12345", command="echo 'HELLO'")
        task = execute_command.run(job)
        assert task['errored_out'] is True

Example #6

Source File: user.py From weibospider with MIT License

5 votes

def crawl_person_infos(uid):
    """
    Crawl user info and their fans and followers
    For the limit of weibo's backend, we can only crawl 5 pages of the fans and followers.
    We also have no permissions to view enterprise's followers and fans info
    :param uid: current user id
    :return: None
    """
    if not uid:
        return

    try:
        user, is_crawled = get_profile(uid)
        # If it's enterprise user, just skip it
        if user and user.verify_type == 2:
            SeedidsOper.set_seed_other_crawled(uid)
            return

        # Crawl fans and followers
        if not is_crawled:
            app.send_task('tasks.user.crawl_follower_fans', args=(uid,), queue='fans_followers',
                          routing_key='for_fans_followers')

    # By adding '--soft-time-limit secs' when you start celery, this will resend task to broker
    # e.g. celery -A tasks.workers -Q user_crawler worker -l info -c 1 --soft-time-limit 10
    except SoftTimeLimitExceeded:
        crawler.error("user SoftTimeLimitExceeded    uid={uid}".format(uid=uid))
        app.send_task('tasks.user.crawl_person_infos', args=(uid, ), queue='user_crawler',
                      routing_key='for_user_info')

Example #7