Python google.appengine.api.taskqueue.Task() Examples

The following are code examples for showing how to use google.appengine.api.taskqueue.Task(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: bbq   Author: ocadotechnology   File: task_creator.py    Apache License 2.0 6 votes vote down vote up
def create_copy_job(cls, copy_job_request):
        task_name = CopyJobTaskName(copy_job_request).create()
        queue_name = copy_job_request.copy_job_type_id + '-copy-job'
        logging.info("Schedule copy_job_task for '%s' in queue '%s'",
                     copy_job_request, queue_name)
        # try:
        task = Task(
            method='POST',
            url='/tasks/copy_job_async/copy_job',
            name=task_name,
            params={"copyJobRequest": json.dumps(copy_job_request,
                                                 cls=RequestEncoder)},
        )
        Tasks.schedule(queue_name, task)

        logging.info('Task %s enqueued, ETA %s', task.name, task.eta) 
Example 2
Project: bbq   Author: ocadotechnology   File: task_creator.py    Apache License 2.0 6 votes vote down vote up
def create_copy_job_result_check(cls, result_check_request):
        assert result_check_request.retry_count >= 0
        queue_name = result_check_request.copy_job_type_id + '-result-check'
        logging.info(
            "Schedule copy_job_result_check task for %s in queue '%s'",
            result_check_request, queue_name)

        task = Task(
            method='POST',
            url='/tasks/copy_job_async/result_check',
            countdown=configuration.copy_job_result_check_countdown_in_sec,
            params={
                "resultCheckRequest": json.dumps(result_check_request,
                                                 cls=RequestEncoder)}
        )
        Tasks.schedule(queue_name, task)

        logging.info('Task %s enqueued, ETA %s', task.name, task.eta) 
Example 3
Project: bbq   Author: ocadotechnology   File: task_creator.py    Apache License 2.0 6 votes vote down vote up
def create_post_copy_action(cls, copy_job_type_id, post_copy_action_request,
                                job_json):
        queue_name = copy_job_type_id + '-post-copy-action'
        logging.info(
            "Creating task on queue '%s' for post copy action ('%s'): %s ",
            queue_name, copy_job_type_id, post_copy_action_request)

        task = Task(
            method='POST',
            url=post_copy_action_request.url,
            payload=json.dumps({
                "data": post_copy_action_request.data,
                "jobJson": job_json
            }, cls=RequestEncoder)
        )
        Tasks.schedule(queue_name, task)

        logging.info('Task %s enqueued, ETA %s', task.name, task.eta) 
Example 4
Project: bbq   Author: ocadotechnology   File: organization_retention.py    Apache License 2.0 6 votes vote down vote up
def schedule_retention_tasks_starting_from_cursor(cls, table_cursor):
        results, next_cursor, more = Table.query() \
            .filter(OrganizationRetention.__table_with_backup_predicate()) \
            .order(Table.last_checked, Table.key) \
            .fetch_page(
            page_size=cls.QUERY_PAGE_SIZE,
            start_cursor=table_cursor
        )
        tasks = [cls.__create_table_retention_task(result)
                 for result in results]
        Tasks.schedule(queue_name='table-retention', tasks=tasks)
        if more and next_cursor:
            task = Task(
                method='GET',
                url='/cron/retention',
                params={
                    'cursor': next_cursor.urlsafe(),
                })

            Tasks.schedule(queue_name='table-retention-scheduler', tasks=[task]) 
Example 5
Project: mlab-ns   Author: m-lab   File: handlers.py    Apache License 2.0 6 votes vote down vote up
def _run_task_hook(hooks, method, task, queue_name):
  """Invokes hooks.method(task, queue_name).

  Args:
    hooks: A hooks.Hooks instance or None.
    method: The name of the method to invoke on the hooks class e.g.
        "enqueue_kickoff_task".
    task: The taskqueue.Task to pass to the hook method.
    queue_name: The name of the queue to pass to the hook method.

  Returns:
    True if the hooks.Hooks instance handled the method, False otherwise.
  """
  if hooks is not None:
    try:
      getattr(hooks, method)(task, queue_name)
    except NotImplementedError:
      # Use the default task addition implementation.
      return False

    return True
  return False 
Example 6
Project: mlab-ns   Author: m-lab   File: handlers.py    Apache License 2.0 6 votes vote down vote up
def schedule(cls, base_path, mapreduce_spec):
    """Schedule finalize task.

    Args:
      mapreduce_spec: mapreduce specification as MapreduceSpec.
    """
    task_name = mapreduce_spec.mapreduce_id + "-finalize"
    finalize_task = taskqueue.Task(
        name=task_name,
        url=base_path + "/finalizejob_callback",
        params={"mapreduce_id": mapreduce_spec.mapreduce_id})
    queue_name = os.environ.get("HTTP_X_APPENGINE_QUEUENAME", "default")
    if not _run_task_hook(mapreduce_spec.get_hooks(),
                          "enqueue_controller_task",
                          finalize_task,
                          queue_name):
      try:
        finalize_task.add(queue_name)
      except (taskqueue.TombstonedTaskError,
              taskqueue.TaskAlreadyExistsError), e:
        logging.warning("Task %r already exists. %s: %s",
                        task_name, e.__class__, e) 
Example 7
Project: mlab-ns   Author: m-lab   File: pipeline.py    Apache License 2.0 6 votes vote down vote up
def get_callback_task(self, *args, **kwargs):
    """Returns a task for calling back this Pipeline.

    Args:
      params: Keyword argument containing a dictionary of key/value pairs
        that will be passed to the callback when it is executed.
      args, kwargs: Passed to the taskqueue.Task constructor. Use these
        arguments to set the task name (for idempotence), etc.

    Returns:
      A taskqueue.Task instance that must be enqueued by the caller.
    """
    if not self.async:
      raise UnexpectedPipelineError(
          'May only call get_callback_task() method for asynchronous pipelines.')

    params = kwargs.get('params', {})
    kwargs['params'] = params
    params['pipeline_id'] = self._pipeline_key.name()
    kwargs['url'] = self.base_path + '/callback'
    kwargs['method'] = 'POST'
    return taskqueue.Task(*args, **kwargs) 
Example 8
Project: mlab-ns   Author: m-lab   File: pipeline.py    Apache License 2.0 6 votes vote down vote up
def cleanup(self):
    """Clean up this Pipeline and all Datastore records used for coordination.

    Only works when called on a root pipeline. Child pipelines will ignore
    calls to this method.

    After this method is called, Pipeline.from_id() and related status
    methods will return inconsistent or missing results. This method is
    fire-and-forget and asynchronous.
    """
    if self._root_pipeline_key is None:
      raise UnexpectedPipelineError(
          'Could not cleanup Pipeline with unknown root pipeline ID.')
    if not self.is_root:
      return
    task = taskqueue.Task(
        params=dict(root_pipeline_key=self._root_pipeline_key),
        url=self.base_path + '/cleanup',
        headers={'X-Ae-Pipeline-Key': self._root_pipeline_key})
    taskqueue.Queue(self.queue_name).add(task) 
Example 9
Project: roger-api   Author: rogertalk   File: jobs.py    MIT License 6 votes vote down vote up
def chat_announce():
    owner_id = int(request.form['owner_id'])
    channel_id = request.form['channel_id']
    text = request.form['text']
    cursor = datastore_query.Cursor(urlsafe=request.form.get('cursor'))
    owner_key = ndb.Key('Account', owner_id)
    owner_future = owner_key.get_async()
    keys, next_cursor, more = models.AccountFollow.fetch_followers_page(
        owner_key, 50, keys_only=True, start_cursor=cursor)
    owner = owner_future.get_result()
    futures = []
    if more:
        task = taskqueue.Task(
            url='/_ah/jobs/chat_announce',
            params={'owner_id': owner_id,
                    'channel_id': channel_id,
                    'text': text,
                    'cursor': next_cursor.urlsafe()},
            retry_options=taskqueue.TaskRetryOptions(task_retry_limit=0))
        futures.append(_add_task_async(task, queue_name=config.INTERNAL_QUEUE))
    logging.debug('Notifying %d/%d followers that %d is on chat: %r',
        len(keys), owner.follower_count, owner_id, text)
    futures.extend(_notify_follower_chat_async(k, owner, channel_id, text) for k in keys)
    _wait_all(futures)
    return '' 
Example 10
Project: roger-api   Author: rogertalk   File: admin.py    MIT License 6 votes vote down vote up
def update_old_youtube_accounts(self):
        self.auto_page = True
        self.page_size = 100
        delay = 1
        futures = []
        q = models.Identity.query()
        q = q.filter(models.Identity.key > ndb.Key('Identity', 'youtube:'))
        q = q.filter(models.Identity.key < ndb.Key('Identity', 'youtube;'))
        accounts = ndb.get_multi([i.account for i in self.run(q) if i.account])
        for account in accounts:
            if account.youtube_channel_id and account.youtube_subs_updated:
                time_since_update = datetime.utcnow() - account.youtube_subs_updated
                if time_since_update < timedelta(days=2):
                    continue
            task = taskqueue.Task(url='/_ah/jobs/update_youtube_channel',
                                  countdown=delay,
                                  params={'account_id': account.key.id()})
            futures.append(self.add_task_async(task, queue_name=config.INTERNAL_QUEUE))
            self.processed_record()
            delay += 1
        self.auto_page_delay = delay
        ndb.Future.wait_all(futures) 
Example 11
Project: roger-api   Author: rogertalk   File: admin.py    MIT License 6 votes vote down vote up
def update_thumbnails_reaction(self):
        self.auto_page = True
        self.auto_page_delay = 10
        self.page_size = 10
        futures = []
        q = models.Content.query(models.Content.tags == 'reaction')
        q = q.order(-models.Content.created)
        content = None
        for content in self.run(q):
            if not content.thumb_url and content.video_url:
                task = taskqueue.Task(url='/_ah/jobs/generate_thumbnail',
                                      params={'content_id': content.key.id()})
                futures.append(self.add_task_async(task, queue_name=config.INTERNAL_QUEUE))
                self.processed_record()
        self.log('Oldest content: %s', content.created)
        ndb.Future.wait_all(futures) 
Example 12
Project: stratego.io   Author: benletchford   File: api.py    MIT License 6 votes vote down vote up
def post(self):
        if not _array_has_values(self.request.arguments(), ['board', 'socket_id']):
            self.response.set_status(STATUS_CODES.INTERNAL_ERROR)
            return

        board = self.request.get('board')
        socket_id = self.request.get('socket_id')

        params = {
            'setup': board,
            'socket_id': socket_id
        }

        q = taskqueue.Queue('pool')
        q.add(
            taskqueue.Task(url='/api/pool/process', params=params, method='post'))

        self.response.set_status(200) 
Example 13
Project: python-docs-samples   Author: GoogleCloudPlatform   File: application.py    Apache License 2.0 6 votes vote down vote up
def post(self):
        amount = int(self.request.get('amount'))

        task = taskqueue.add(
            url='/update_counter',
            target='worker',
            params={'amount': amount})

        self.response.write(
            'Task {} enqueued, ETA {}.'.format(task.name, task.eta))


# AsyncEnqueueTaskHandler behaves the same as EnqueueTaskHandler, but shows
# how to queue the task using the asyncronous API. This is not wired up by
# default. To use this, change the MainPageHandler's form action to
# /enqueue_async 
Example 14
Project: python-docs-samples   Author: GoogleCloudPlatform   File: application.py    Apache License 2.0 6 votes vote down vote up
def post(self):
        amount = int(self.request.get('amount'))

        queue = taskqueue.Queue(name='default')
        task = taskqueue.Task(
            url='/update_counter',
            target='worker',
            params={'amount': amount})

        rpc = queue.add_async(task)

        # Wait for the rpc to complete and return the queued task.
        task = rpc.get_result()

        self.response.write(
            'Task {} enqueued, ETA {}.'.format(task.name, task.eta)) 
Example 15
Project: luci-py   Author: luci   File: pipeline.py    Apache License 2.0 6 votes vote down vote up
def get_callback_task(self, *args, **kwargs):
    """Returns a task for calling back this Pipeline.

    Args:
      params: Keyword argument containing a dictionary of key/value pairs
        that will be passed to the callback when it is executed.
      args, kwargs: Passed to the taskqueue.Task constructor. Use these
        arguments to set the task name (for idempotence), etc.

    Returns:
      A taskqueue.Task instance that must be enqueued by the caller.
    """
    if not self.async:
      raise UnexpectedPipelineError(
          'May only call get_callback_task() method for asynchronous pipelines.')

    params = kwargs.get('params', {})
    kwargs['params'] = params
    params['pipeline_id'] = self._pipeline_key.name()
    kwargs['url'] = self.base_path + '/callback'
    kwargs['method'] = 'POST'
    return taskqueue.Task(*args, **kwargs) 
Example 16
Project: luci-py   Author: luci   File: pipeline.py    Apache License 2.0 6 votes vote down vote up
def cleanup(self):
    """Clean up this Pipeline and all Datastore records used for coordination.

    Only works when called on a root pipeline. Child pipelines will ignore
    calls to this method.

    After this method is called, Pipeline.from_id() and related status
    methods will return inconsistent or missing results. This method is
    fire-and-forget and asynchronous.
    """
    if self._root_pipeline_key is None:
      raise UnexpectedPipelineError(
          'Could not cleanup Pipeline with unknown root pipeline ID.')
    if not self.is_root:
      return
    task = taskqueue.Task(
        params=dict(root_pipeline_key=self._root_pipeline_key),
        url=self.base_path + '/cleanup',
        headers={'X-Ae-Pipeline-Key': self._root_pipeline_key})
    taskqueue.Queue(self.queue_name).add(task) 
Example 17
Project: luci-py   Author: luci   File: map_job_control.py    Apache License 2.0 6 votes vote down vote up
def __add_kickoff_task(cls, job_config, mapreduce_spec):
    """Add kickoff task to taskqueue.

    Args:
      job_config: map_job.JobConfig.
      mapreduce_spec: model.MapreduceSpec,
    """
    params = {"mapreduce_id": job_config.job_id}
    # Task is not named so that it can be added within a transaction.
    kickoff_task = taskqueue.Task(
        # TODO(user): Perhaps make this url a computed field of job_config.
        url=job_config._base_path + "/kickoffjob_callback/" + job_config.job_id,
        headers=util._get_task_headers(job_config.job_id),
        params=params)
    if job_config._hooks_cls:
      hooks = job_config._hooks_cls(mapreduce_spec)
      try:
        hooks.enqueue_kickoff_task(kickoff_task, job_config.queue_name)
        return
      except NotImplementedError:
        pass
    kickoff_task.add(job_config.queue_name, transactional=True) 
Example 18
Project: luci-py   Author: luci   File: model.py    Apache License 2.0 6 votes vote down vote up
def decode_payload(cls, request):
    """Decode task payload.

    HugeTask controls its own payload entirely including urlencoding.
    It doesn't depend on any particular web framework.

    Args:
      request: a webapp Request instance.

    Returns:
      A dict of str to str. The same as the params argument to __init__.

    Raises:
      DeprecationWarning: When task payload constructed from an older
        incompatible version of mapreduce.
    """
    # TODO(user): Pass mr_id into headers. Otherwise when payload decoding
    # failed, we can't abort a mr.
    if request.headers.get(cls.PAYLOAD_VERSION_HEADER) != cls.PAYLOAD_VERSION:
      raise DeprecationWarning(
          "Task is generated by an older incompatible version of mapreduce. "
          "Please kill this job manually")
    return cls._decode_payload(request.body) 
Example 19
Project: luci-py   Author: luci   File: handlers.py    Apache License 2.0 6 votes vote down vote up
def _run_task_hook(hooks, method, task, queue_name):
  """Invokes hooks.method(task, queue_name).

  Args:
    hooks: A hooks.Hooks instance or None.
    method: The name of the method to invoke on the hooks class e.g.
        "enqueue_kickoff_task".
    task: The taskqueue.Task to pass to the hook method.
    queue_name: The name of the queue to pass to the hook method.

  Returns:
    True if the hooks.Hooks instance handled the method, False otherwise.
  """
  if hooks is not None:
    try:
      getattr(hooks, method)(task, queue_name)
    except NotImplementedError:
      # Use the default task addition implementation.
      return False

    return True
  return False 
Example 20
Project: luci-py   Author: luci   File: handlers.py    Apache License 2.0 6 votes vote down vote up
def _check_mr_state(cls, state, mr_id):
    """Check MapreduceState.

    Args:
      state: an MapreduceState instance.
      mr_id: mapreduce id.

    Returns:
      True if state is valid. False if not and this task should be dropped.
    """
    if state is None:
      logging.warning(
          "Mapreduce State for job %s is missing. Dropping Task.",
          mr_id)
      return False
    if not state.active:
      logging.warning(
          "Mapreduce %s is not active. Looks like spurious task "
          "execution. Dropping Task.", mr_id)
      return False
    return True 
Example 21
Project: luci-py   Author: luci   File: handlers.py    Apache License 2.0 6 votes vote down vote up
def _add_kickoff_task(cls,
                        base_path,
                        mapreduce_spec,
                        eta,
                        countdown,
                        queue_name):
    """Enqueues a new kickoff task."""
    params = {"mapreduce_id": mapreduce_spec.mapreduce_id}
    # Task is not named so that it can be added within a transaction.
    kickoff_task = taskqueue.Task(
        url=base_path + "/kickoffjob_callback/" + mapreduce_spec.mapreduce_id,
        headers=util._get_task_headers(mapreduce_spec.mapreduce_id),
        params=params,
        eta=eta,
        countdown=countdown)
    hooks = mapreduce_spec.get_hooks()
    if hooks is not None:
      try:
        hooks.enqueue_kickoff_task(kickoff_task, queue_name)
        return
      except NotImplementedError:
        pass
    kickoff_task.add(queue_name, transactional=True) 
Example 22
Project: luci-py   Author: luci   File: handlers.py    Apache License 2.0 6 votes vote down vote up
def schedule(cls, mapreduce_spec):
    """Schedule finalize task.

    Args:
      mapreduce_spec: mapreduce specification as MapreduceSpec.
    """
    task_name = mapreduce_spec.mapreduce_id + "-finalize"
    finalize_task = taskqueue.Task(
        name=task_name,
        url=(mapreduce_spec.params["base_path"] + "/finalizejob_callback/" +
             mapreduce_spec.mapreduce_id),
        params={"mapreduce_id": mapreduce_spec.mapreduce_id},
        headers=util._get_task_headers(mapreduce_spec.mapreduce_id))
    queue_name = util.get_queue_name(None)
    if not _run_task_hook(mapreduce_spec.get_hooks(),
                          "enqueue_controller_task",
                          finalize_task,
                          queue_name):
      try:
        finalize_task.add(queue_name)
      except (taskqueue.TombstonedTaskError,
              taskqueue.TaskAlreadyExistsError), e:
        logging.warning("Task %r already exists. %s: %s",
                        task_name, e.__class__, e) 
Example 23
Project: luci-py   Author: luci   File: pipeline.py    Apache License 2.0 6 votes vote down vote up
def get_callback_task(self, *args, **kwargs):
    """Returns a task for calling back this Pipeline.

    Args:
      params: Keyword argument containing a dictionary of key/value pairs
        that will be passed to the callback when it is executed.
      args, kwargs: Passed to the taskqueue.Task constructor. Use these
        arguments to set the task name (for idempotence), etc.

    Returns:
      A taskqueue.Task instance that must be enqueued by the caller.
    """
    if not self.async:
      raise UnexpectedPipelineError(
          'May only call get_callback_task() method for asynchronous pipelines.')

    params = kwargs.get('params', {})
    kwargs['params'] = params
    params['pipeline_id'] = self._pipeline_key.name()
    kwargs['url'] = self.base_path + '/callback'
    kwargs['method'] = 'POST'
    return taskqueue.Task(*args, **kwargs) 
Example 24
Project: luci-py   Author: luci   File: pipeline.py    Apache License 2.0 6 votes vote down vote up
def cleanup(self):
    """Clean up this Pipeline and all Datastore records used for coordination.

    Only works when called on a root pipeline. Child pipelines will ignore
    calls to this method.

    After this method is called, Pipeline.from_id() and related status
    methods will return inconsistent or missing results. This method is
    fire-and-forget and asynchronous.
    """
    if self._root_pipeline_key is None:
      raise UnexpectedPipelineError(
          'Could not cleanup Pipeline with unknown root pipeline ID.')
    if not self.is_root:
      return
    task = taskqueue.Task(
        params=dict(root_pipeline_key=self._root_pipeline_key),
        url=self.base_path + '/cleanup',
        headers={'X-Ae-Pipeline-Key': self._root_pipeline_key})
    taskqueue.Queue(self.queue_name).add(task) 
Example 25
Project: luci-py   Author: luci   File: map_job_control.py    Apache License 2.0 6 votes vote down vote up
def __add_kickoff_task(cls, job_config, mapreduce_spec):
    """Add kickoff task to taskqueue.

    Args:
      job_config: map_job.JobConfig.
      mapreduce_spec: model.MapreduceSpec,
    """
    params = {"mapreduce_id": job_config.job_id}
    # Task is not named so that it can be added within a transaction.
    kickoff_task = taskqueue.Task(
        # TODO(user): Perhaps make this url a computed field of job_config.
        url=job_config._base_path + "/kickoffjob_callback/" + job_config.job_id,
        headers=util._get_task_headers(job_config.job_id),
        params=params)
    if job_config._hooks_cls:
      hooks = job_config._hooks_cls(mapreduce_spec)
      try:
        hooks.enqueue_kickoff_task(kickoff_task, job_config.queue_name)
        return
      except NotImplementedError:
        pass
    kickoff_task.add(job_config.queue_name, transactional=True) 
Example 26
Project: luci-py   Author: luci   File: model.py    Apache License 2.0 6 votes vote down vote up
def decode_payload(cls, request):
    """Decode task payload.

    HugeTask controls its own payload entirely including urlencoding.
    It doesn't depend on any particular web framework.

    Args:
      request: a webapp Request instance.

    Returns:
      A dict of str to str. The same as the params argument to __init__.

    Raises:
      DeprecationWarning: When task payload constructed from an older
        incompatible version of mapreduce.
    """
    # TODO(user): Pass mr_id into headers. Otherwise when payload decoding
    # failed, we can't abort a mr.
    if request.headers.get(cls.PAYLOAD_VERSION_HEADER) != cls.PAYLOAD_VERSION:
      raise DeprecationWarning(
          "Task is generated by an older incompatible version of mapreduce. "
          "Please kill this job manually")
    return cls._decode_payload(request.body) 
Example 27
Project: luci-py   Author: luci   File: handlers.py    Apache License 2.0 6 votes vote down vote up
def _run_task_hook(hooks, method, task, queue_name):
  """Invokes hooks.method(task, queue_name).

  Args:
    hooks: A hooks.Hooks instance or None.
    method: The name of the method to invoke on the hooks class e.g.
        "enqueue_kickoff_task".
    task: The taskqueue.Task to pass to the hook method.
    queue_name: The name of the queue to pass to the hook method.

  Returns:
    True if the hooks.Hooks instance handled the method, False otherwise.
  """
  if hooks is not None:
    try:
      getattr(hooks, method)(task, queue_name)
    except NotImplementedError:
      # Use the default task addition implementation.
      return False

    return True
  return False 
Example 28
Project: luci-py   Author: luci   File: handlers.py    Apache License 2.0 6 votes vote down vote up
def _check_mr_state(cls, state, mr_id):
    """Check MapreduceState.

    Args:
      state: an MapreduceState instance.
      mr_id: mapreduce id.

    Returns:
      True if state is valid. False if not and this task should be dropped.
    """
    if state is None:
      logging.warning(
          "Mapreduce State for job %s is missing. Dropping Task.",
          mr_id)
      return False
    if not state.active:
      logging.warning(
          "Mapreduce %s is not active. Looks like spurious task "
          "execution. Dropping Task.", mr_id)
      return False
    return True 
Example 29
Project: luci-py   Author: luci   File: handlers.py    Apache License 2.0 6 votes vote down vote up
def _add_kickoff_task(cls,
                        base_path,
                        mapreduce_spec,
                        eta,
                        countdown,
                        queue_name):
    """Enqueues a new kickoff task."""
    params = {"mapreduce_id": mapreduce_spec.mapreduce_id}
    # Task is not named so that it can be added within a transaction.
    kickoff_task = taskqueue.Task(
        url=base_path + "/kickoffjob_callback/" + mapreduce_spec.mapreduce_id,
        headers=util._get_task_headers(mapreduce_spec.mapreduce_id),
        params=params,
        eta=eta,
        countdown=countdown)
    hooks = mapreduce_spec.get_hooks()
    if hooks is not None:
      try:
        hooks.enqueue_kickoff_task(kickoff_task, queue_name)
        return
      except NotImplementedError:
        pass
    kickoff_task.add(queue_name, transactional=True) 
Example 30
Project: luci-py   Author: luci   File: handlers.py    Apache License 2.0 6 votes vote down vote up
def schedule(cls, mapreduce_spec):
    """Schedule finalize task.

    Args:
      mapreduce_spec: mapreduce specification as MapreduceSpec.
    """
    task_name = mapreduce_spec.mapreduce_id + "-finalize"
    finalize_task = taskqueue.Task(
        name=task_name,
        url=(mapreduce_spec.params["base_path"] + "/finalizejob_callback/" +
             mapreduce_spec.mapreduce_id),
        params={"mapreduce_id": mapreduce_spec.mapreduce_id},
        headers=util._get_task_headers(mapreduce_spec.mapreduce_id))
    queue_name = util.get_queue_name(None)
    if not _run_task_hook(mapreduce_spec.get_hooks(),
                          "enqueue_controller_task",
                          finalize_task,
                          queue_name):
      try:
        finalize_task.add(queue_name)
      except (taskqueue.TombstonedTaskError,
              taskqueue.TaskAlreadyExistsError), e:
        logging.warning("Task %r already exists. %s: %s",
                        task_name, e.__class__, e) 
Example 31
Project: bbq   Author: ocadotechnology   File: organization_retention.py    Apache License 2.0 5 votes vote down vote up
def __create_table_retention_task(table):
        params = {'projectId': table.project_id,
                  'datasetId': table.dataset_id,
                  'tableId': table.table_id,
                  'tableKey': table.key.urlsafe()}
        if table.partition_id:
            params['partitionId'] = table.partition_id
        return Task(
            method='GET',
            url='/tasks/retention/table',
            params=params) 
Example 32
Project: bbq   Author: ocadotechnology   File: test_tasks.py    Apache License 2.0 5 votes vote down vote up
def test_schedule_can_work_with_single_task(self):
        # given
        task1 = Task(
            url='/example/task1',
        )
        # when
        Tasks.schedule("default", task1)
        # then
        executed_tasks = self.taskqueue_stub.get_filtered_tasks(
            queue_names="default"
        )

        self.assertEqual(len(executed_tasks), 1, "Should create one task in queue")
        self.assertEqual(executed_tasks[0].url, '/example/task1') 
Example 33
Project: mlab-ns   Author: m-lab   File: util.py    Apache License 2.0 5 votes vote down vote up
def add(self, queue_name, transactional=False, parent=None):
    """Add task to the queue."""
    payload_str = urllib.urlencode(self.params)
    if len(payload_str) < self.MAX_TASK_PAYLOAD:
      # Payload is small. Don't bother with anything.
      task = self.to_task()
      task.add(queue_name, transactional)
      return

    compressed_payload = base64.b64encode(zlib.compress(payload_str))

    if len(compressed_payload) < self.MAX_TASK_PAYLOAD:
      # Compressed payload is small. Don't bother with datastore.
      task = taskqueue.Task(
          url=self.url,
          params={self.PAYLOAD_PARAM: compressed_payload},
          name=self.name,
          eta=self.eta,
          countdown=self.countdown)
      task.add(queue_name, transactional)
      return

    if len(compressed_payload) > self.MAX_DB_PAYLOAD:
      raise Exception("Payload to big to be stored in database: %s",
                      len(compressed_payload))

    # Store payload in the datastore.
    if not parent:
      raise Exception("Huge tasks should specify parent entity.")

    payload_entity = _HugeTaskPayload(payload=compressed_payload,
                                      parent=parent)

    payload_key = payload_entity.put()
    task = taskqueue.Task(
        url=self.url,
        params={self.PAYLOAD_KEY_PARAM: str(payload_key)},
        name=self.name,
        eta=self.eta,
        countdown=self.countdown)
    task.add(queue_name, transactional) 
Example 34
Project: mlab-ns   Author: m-lab   File: util.py    Apache License 2.0 5 votes vote down vote up
def to_task(self):
    """Convert to a taskqueue task without doing any kind of encoding."""
    return taskqueue.Task(
        url=self.url,
        params=self.params,
        name=self.name,
        eta=self.eta,
        countdown=self.countdown) 
Example 35
Project: mlab-ns   Author: m-lab   File: handlers.py    Apache License 2.0 5 votes vote down vote up
def _finalize_job(mapreduce_spec, mapreduce_state, base_path):
    """Finalize job execution.

    Finalizes output writer, invokes done callback an schedules
    finalize job execution.

    Args:
      mapreduce_spec: an instance of MapreduceSpec
      mapreduce_state: an instance of MapreduceState
      base_path: handler base path.
    """
    config = util.create_datastore_write_config(mapreduce_spec)
    # Enqueue done_callback if needed.
    if mapreduce_spec.mapper.output_writer_class():
      mapreduce_spec.mapper.output_writer_class().finalize_job(mapreduce_state)
    def put_state(state):
      state.put(config=config)
      done_callback = mapreduce_spec.params.get(
          model.MapreduceSpec.PARAM_DONE_CALLBACK)
      if done_callback:
        done_task = taskqueue.Task(
            url=done_callback,
            headers={"Mapreduce-Id": mapreduce_spec.mapreduce_id},
            method=mapreduce_spec.params.get("done_callback_method", "POST"))
        queue_name = mapreduce_spec.params.get(
            model.MapreduceSpec.PARAM_DONE_CALLBACK_QUEUE,
            "default")

        if not _run_task_hook(mapreduce_spec.get_hooks(),
                              "enqueue_done_task",
                              done_task,
                              queue_name):
          done_task.add(queue_name, transactional=True)
      FinalizeJobHandler.schedule(base_path, mapreduce_spec)

    db.run_in_transaction(put_state, mapreduce_state) 
Example 36
Project: mlab-ns   Author: m-lab   File: pipeline.py    Apache License 2.0 5 votes vote down vote up
def post(self):
    if 'HTTP_X_APPENGINE_TASKNAME' not in self.request.environ:
      self.response.set_status(403)
      return

    context = _PipelineContext.from_environ(self.request.environ)

    # Set of stringified db.Keys of children to run.
    all_pipeline_keys = set()

    # For backwards compatibility with the old style of fan-out requests.
    all_pipeline_keys.update(self.request.get_all('pipeline_key'))

    # Fetch the child pipelines from the parent. This works around the 10KB
    # task payload limit. This get() is consistent-on-read and the fan-out
    # task is enqueued in the transaction that updates the parent, so the
    # fanned_out property is consistent here.
    parent_key = self.request.get('parent_key')
    child_indexes = [int(x) for x in self.request.get_all('child_indexes')]
    if parent_key:
      parent_key = db.Key(parent_key)
      parent = db.get(parent_key)
      for index in child_indexes:
        all_pipeline_keys.add(str(parent.fanned_out[index]))

    all_tasks = []
    for pipeline_key in all_pipeline_keys:
      all_tasks.append(taskqueue.Task(
          url=context.pipeline_handler_path,
          params=dict(pipeline_key=pipeline_key),
          headers={'X-Ae-Pipeline-Key': pipeline_key},
          name='ae-pipeline-fan-out-' + db.Key(pipeline_key).name()))

    batch_size = 100  # Limit of taskqueue API bulk add.
    for i in xrange(0, len(all_tasks), batch_size):
      batch = all_tasks[i:i+batch_size]
      try:
        taskqueue.Queue(context.queue_name).add(batch)
      except (taskqueue.TombstonedTaskError, taskqueue.TaskAlreadyExistsError):
        pass 
Example 37
Project: gcp-census   Author: ocadotechnology   File: bigquery_task.py    Apache License 2.0 5 votes vote down vote up
def schedule_task_for_each_table(self, project_id, dataset_id,
                                     page_token=None):
        list_response = self.big_query.list_tables(project_id, dataset_id,
                                                   page_token=page_token)
        if 'tables' in list_response:
            table_id_list = [table['tableReference']['tableId']
                             for table in list_response['tables']]
            tasks = self.create_table_tasks(project_id, dataset_id,
                                            table_id_list)
            Tasks.schedule(queue_name='bigquery-tables', tasks=tasks)
        else:
            logging.info("Dataset %s.%s is empty", project_id, dataset_id)
            return

        if 'nextPageToken' in list_response:
            url = '/bigQuery/project/%s/dataset/%s?pageToken=%s' % (
                project_id, dataset_id, list_response['nextPageToken'])
            task_name = '%s-%s-%s-%s' % (project_id, dataset_id,
                                         list_response['nextPageToken'],
                                         datetime.utcnow().strftime("%Y%m%d"))
            next_task = Task(
                method='GET',
                url=url,
                name=task_name)
            Tasks.schedule(queue_name='bigquery-list', tasks=[next_task])
        else:
            logging.info("There is no more tables in this dataset") 
Example 38
Project: gcp-census   Author: ocadotechnology   File: bigquery_task.py    Apache License 2.0 5 votes vote down vote up
def create_project_tasks(project_id_list):
        for project_id in project_id_list:
            yield Task(method='GET',
                       url='/bigQuery/project/%s' % project_id) 
Example 39
Project: gcp-census   Author: ocadotechnology   File: bigquery_task.py    Apache License 2.0 5 votes vote down vote up
def create_dataset_tasks(project_id, dataset_id_list):
        for dataset_id in dataset_id_list:
            yield Task(method='GET',
                       url='/bigQuery/project/%s/dataset/%s'
                           % (project_id, dataset_id)) 
Example 40
Project: gcp-census   Author: ocadotechnology   File: bigquery_task.py    Apache License 2.0 5 votes vote down vote up
def create_table_tasks(project_id, dataset_id, table_id_list):
        for table_id in table_id_list:
            yield Task(method='GET',
                       url='/bigQuery/project/%s/dataset/%s/table/%s'
                           % (project_id, dataset_id, table_id)) 
Example 41
Project: appengine-python-flask-travis-ci   Author: russomi   File: test_task_queue.py    Apache License 2.0 5 votes vote down vote up
def testTaskAddedToQueue(self):
        taskqueue.Task(name='my_task', url='/url/of/my/task/').add()
        tasks = self.taskqueue_stub.get_filtered_tasks()
        assert len(tasks) == 1
        assert tasks[0].name == 'my_task'
# [END taskqueue]

    # [START filtering] 
Example 42
Project: appengine-python-flask-travis-ci   Author: russomi   File: test_task_queue.py    Apache License 2.0 5 votes vote down vote up
def testFiltering(self):
        taskqueue.Task(name='task_one', url='/url/of/task/1/').add('queue-1')
        taskqueue.Task(name='task_two', url='/url/of/task/2/').add('queue-2')

        # All tasks
        tasks = self.taskqueue_stub.get_filtered_tasks()
        assert len(tasks) == 2

        # Filter by name
        tasks = self.taskqueue_stub.get_filtered_tasks(name='task_one')
        assert len(tasks) == 1
        assert tasks[0].name == 'task_one'

        # Filter by URL
        tasks = self.taskqueue_stub.get_filtered_tasks(url='/url/of/task/1/')
        assert len(tasks) == 1
        assert tasks[0].name == 'task_one'

        # Filter by queue
        tasks = self.taskqueue_stub.get_filtered_tasks(queue_names='queue-1')
        assert len(tasks) == 1
        assert tasks[0].name == 'task_one'

        # Multiple queues
        tasks = self.taskqueue_stub.get_filtered_tasks(
            queue_names=['queue-1', 'queue-2'])
        assert len(tasks) == 2
    # [END filtering]

    # [START deferred] 
Example 43
Project: roger-api   Author: rogertalk   File: report.py    MIT License 5 votes vote down vote up
def report_async(self, event):
        row = vars(event)
        # Create an insert id for the event so that double runs won't insert it twice.
        row[bigquery_api.INSERT_ID_KEY] = random.base62(10)
        task = taskqueue.Task(
            method='PULL',
            tag=event.name,
            payload=bigquery_api.json_encoder.encode(row))
        return task.add_async(queue_name=config.BIGQUERY_QUEUE_NAME) 
Example 44
Project: roger-api   Author: rogertalk   File: cron.py    MIT License 5 votes vote down vote up
def report_to_bigquery():
    """Flush all pending events of a certain type to BigQuery."""
    # Schedule multiple flush jobs per minute for some events.
    if request.method == 'GET':
        tasks = []
        for delay in xrange(0, 60, 5):
            tasks.append(taskqueue.Task(method='POST', url=request.path,
                                        countdown=delay,
                                        params={'event_name': 'content_vote_v1'}))
        tasks.append(taskqueue.Task(method='POST', url=request.path))
        taskqueue.Queue(config.BIGQUERY_CRON_QUEUE_NAME).add(tasks)
        return ''
    # Retrieve pending events from pull queue.
    try:
        q = taskqueue.Queue(config.BIGQUERY_QUEUE_NAME)
        tasks = q.lease_tasks_by_tag(config.BIGQUERY_LEASE_TIME.total_seconds(),
                                     config.BIGQUERY_LEASE_AMOUNT,
                                     tag=flask_extras.get_parameter('event_name'))
        logging.debug('Leased %d event(s) from %s', len(tasks), config.BIGQUERY_QUEUE_NAME)
    except taskqueue.TransientError:
        logging.warning('Could not lease events due to transient error')
        return '', 503
    if not tasks:
        return ''
    # Insert the events into BigQuery.
    table_id = tasks[0].tag
    rows = [json.loads(t.payload) for t in tasks]
    bigquery_client.insert_rows(table_id, rows)
    # Delete the tasks now that we're done with them.
    q.delete_tasks(tasks)
    return '' 
Example 45
Project: roger-api   Author: rogertalk   File: cron.py    MIT License 5 votes vote down vote up
def update_content_requests():
    tags = {'approved', 'default'}
    q = models.ContentRequestPublic.query()
    for tag in tags:
        q = q.filter(models.ContentRequestPublic.tags == tag)
    q = q.order(-models.ContentRequestPublic.sort_index)
    futures = []
    delay = 0
    for request in q:
        # TODO: Improved condition to avoid checking depleted requests.
        # TODO: Add way to manually update content request entries when request is closed.
        if not request.wallet or request.closed:
            continue
        task = taskqueue.Task(
            countdown=delay,
            url='/_ah/jobs/update_content_request_entries',
            params={
                'request_id': str(request.key.id()),
                'wallet_id': str(request.wallet.id()),
                'wallet_owner_id': str(request.wallet_owner.id()),
            },
            retry_options=taskqueue.TaskRetryOptions(task_retry_limit=0))
        futures.append(_add_task_async(task, queue_name=config.INTERNAL_QUEUE))
        delay += 5
    _wait_all(futures)
    logging.debug('Scheduled %d job(s) to update content request entries', len(futures))
    return '' 
Example 46
Project: roger-api   Author: rogertalk   File: cron.py    MIT License 5 votes vote down vote up
def update_youtube_stats():
    futures = []
    delay = 0
    for row in bigquery_client.query(QUERY_REACTED_CONTENT).rows():
        task = taskqueue.Task(
            countdown=delay,
            url='/_ah/jobs/update_youtube_views_batched',
            params={'original_id': row.content_id},
            retry_options=taskqueue.TaskRetryOptions(task_retry_limit=0))
        futures.append(_add_task_async(task, queue_name=config.INTERNAL_QUEUE))
        delay += 2
    _wait_all(futures)
    return '' 
Example 47
Project: roger-api   Author: rogertalk   File: jobs.py    MIT License 5 votes vote down vote up
def _schedule_requests_async(account):
    cache_key = 'onboarding_request_content_ids'
    futures = []
    # Get or create a request pool to sample requests from.
    context = ndb.get_context()
    pool = yield context.memcache_get(cache_key)
    if not pool:
        q = models.Content.query()
        q = q.filter(models.Content.tags == 'original')
        q = q.filter(models.Content.tags == 'is suggestion')
        q = q.order(-models.Content.sort_index)
        content_keys = yield q.fetch_async(50, keys_only=True)
        pool = [k.id() for k in content_keys]
        futures.append(context.memcache_set(cache_key, pool, time=43200))
    if len(pool) < config.ONBOARDING_REQUESTS_COUNT:
        logging.error('Failed to get a content pool of %d or more ids', config.ONBOARDING_REQUESTS_COUNT)
        return
    # First, pick any hard coded content ids from config.
    cids = list(config.ONBOARDING_REQUESTS_INJECT_IDS)
    random.shuffle(cids)
    # Then pick random requests from the request pool.
    cids += [cid for cid in random.sample(pool, config.ONBOARDING_REQUESTS_COUNT) if cid not in cids]
    # Limit number of requests.
    cids = cids[:config.ONBOARDING_REQUESTS_COUNT]
    logging.debug('Picked %r as onboarding requests', cids)
    # Send all requests with a delay (first one immediately; rest twice per day).
    reactioncam = ndb.Key('Account', config.REACTION_CAM_ID)
    for i, cid in enumerate(cids):
        task = taskqueue.Task(
            url='/_ah/jobs/content_request_send',
            countdown=config.ONBOARDING_REQUESTS_CADENCE(i),
            params={'requester_id': reactioncam.id(),
                    'account_id': account.key.id(),
                    'content_id': cid})
        futures.append(_add_task_async(task, queue_name=config.INTERNAL_QUEUE))
    yield tuple(futures) 
Example 48
Project: roger-api   Author: rogertalk   File: api.py    MIT License 5 votes vote down vote up
def _handle_content_became_public(creator, content, related_to):
    event_v1 = events.ContentV1(
        account_key=creator.key,
        content_id=content.key.id(),
        related_id=content.related_to.id() if content.related_to else None,
        duration=content.duration / 1000,
        tags=content.tags)
    event_v2 = events.ContentV2(
        account=creator,
        content_id=content.key.id(),
        related_id=content.related_to.id() if content.related_to else None,
        duration=content.duration / 1000,
        tags=content.tags)
    futures = [event_v1.report_async(), event_v2.report_async()]
    # Notify original creator that their video was referenced.
    if related_to and 'reaction' in related_to.tags:
        # TODO: Consider if we should send this for non-reaction videos.
        handler = accounts.get_handler(related_to.creator)
        logging.debug('Notifying %s that their content was referenced by %s',
            handler.username, creator.username)
        future = handler.notifs.emit_async(
            notifs.ON_CONTENT_REFERENCED,
            content=content, creator=creator)
        futures.append(future)
    # Kick off a job that will notify any users that want to know about this reaction.
    task = taskqueue.Task(
        url='/_ah/jobs/content_became_public',
        params={'content_id': content.key.id(),
                'creator_id': creator.key.id(),
                'original_id': related_to.key.id() if related_to else ''})
    futures.append(_add_task_async(task, queue_name=config.INTERNAL_QUEUE))
    _wait_all(futures) 
Example 49
Project: roger-api   Author: rogertalk   File: api.py    MIT License 5 votes vote down vote up
def _handle_content_was_deleted(creator, content, related_to):
    task = taskqueue.Task(
        countdown=5,
        url='/_ah/jobs/content_was_deleted',
        params={'content_id': content.key.id(),
                'creator_id': creator.key.id(),
                'original_id': related_to.key.id() if related_to else ''})
    task.add(queue_name=config.INTERNAL_QUEUE) 
Example 50
Project: roger-api   Author: rogertalk   File: admin.py    MIT License 5 votes vote down vote up
def post_notify():
    params = {
        'app': request.form['app'],
        'env': request.form['env'],
        'text': request.form['text'],
    }
    title = request.form.get('title')
    if title:
        params['title'] = title
    tokens = re.split(r'\s+', request.files['tokens'].read().strip())
    tasks = []
    total_tasks = 0
    i, j = 0, 500
    while True:
        batch = tokens[i:j]
        if not batch:
            break
        i, j = j, j + j - i
        params['token'] = batch
        tasks.append(taskqueue.Task(method='POST', url='/_ah/jobs/notify_batch',
                                    params=params))
        if len(tasks) == taskqueue.MAX_TASKS_PER_ADD:
            taskqueue.Queue().add(tasks)
            total_tasks += len(tasks)
            tasks = []
    if tasks:
        taskqueue.Queue().add(tasks)
        total_tasks += len(tasks)
    return 'Notifying %d token(s) in %d job(s) • <a href="/admin/notify">Back</a>' % (
            len(tokens),
            total_tasks) 
Example 51
Project: naziscore   Author: rbanffy   File: scoring.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def delete_from_scoring_indirect(task_name):
    logging.debug('Deleting task {} from scoring-indirect'.format(
        task_name))
    return taskqueue.Queue('scoring-indirect').delete_tasks(
        taskqueue.Task(name=task_name)) 
Example 52
Project: server   Author: viur-framework   File: tasks.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def CallableTask( fn ):
	"""Marks a Class as representing a user-callable Task.
	It *should* extend CallableTaskBase and *must* provide
	its API
	"""
	global _callableTasks
	_callableTasks[ fn.key ] = fn
	return( fn ) 
Example 53
Project: python-docs-samples   Author: GoogleCloudPlatform   File: main.py    Apache License 2.0 5 votes vote down vote up
def post(self):
        key = self.request.get('key')
        if key:
            queue = taskqueue.Queue('pullq')
            queue.add(taskqueue.Task(payload='', method='PULL', tag=key))
        self.redirect('/')
    # [END adding_task] 
Example 54
Project: python-docs-samples   Author: GoogleCloudPlatform   File: task_queue_test.py    Apache License 2.0 5 votes vote down vote up
def testTaskAddedToQueue(self):
        taskqueue.Task(name='my_task', url='/url/of/my/task/').add()
        tasks = self.taskqueue_stub.get_filtered_tasks()
        self.assertEqual(len(tasks), 1)
        self.assertEqual(tasks[0].name, 'my_task')
# [END taskqueue]

    # [START filtering] 
Example 55
Project: python-docs-samples   Author: GoogleCloudPlatform   File: task_queue_test.py    Apache License 2.0 5 votes vote down vote up
def testFiltering(self):
        taskqueue.Task(name='task_one', url='/url/of/task/1/').add('queue-1')
        taskqueue.Task(name='task_two', url='/url/of/task/2/').add('queue-2')

        # All tasks
        tasks = self.taskqueue_stub.get_filtered_tasks()
        self.assertEqual(len(tasks), 2)

        # Filter by name
        tasks = self.taskqueue_stub.get_filtered_tasks(name='task_one')
        self.assertEqual(len(tasks), 1)
        self.assertEqual(tasks[0].name, 'task_one')

        # Filter by URL
        tasks = self.taskqueue_stub.get_filtered_tasks(url='/url/of/task/1/')
        self.assertEqual(len(tasks), 1)
        self.assertEqual(tasks[0].name, 'task_one')

        # Filter by queue
        tasks = self.taskqueue_stub.get_filtered_tasks(queue_names='queue-1')
        self.assertEqual(len(tasks), 1)
        self.assertEqual(tasks[0].name, 'task_one')

        # Multiple queues
        tasks = self.taskqueue_stub.get_filtered_tasks(
            queue_names=['queue-1', 'queue-2'])
        self.assertEqual(len(tasks), 2)
    # [END filtering]

    # [START deferred] 
Example 56
Project: andris-projeto   Author: andris210296   File: gaeutil.py    MIT License 5 votes vote down vote up
def __init__(self, queue_name, url, **kwargs):
        '''
        kwargs are the same used on Task class
        (https://developers.google.com/appengine/docs/python/taskqueue/tasks#Task)
        '''
        super(TaskQueueCommand, self).__init__()
        self._task = Task(url=url, **kwargs)
        self._queue_name = queue_name 
Example 57
Project: andris-projeto   Author: andris210296   File: gaeutil.py    MIT License 5 votes vote down vote up
def __init__(self, queue_name, url, **kwargs):
        '''
        kwargs are the same used on Task class
        (https://developers.google.com/appengine/docs/python/taskqueue/tasks#Task)
        '''
        super(TaskQueueCommand, self).__init__()
        self._task = Task(url=url, **kwargs)
        self._queue_name = queue_name 
Example 58
Project: luci-py   Author: luci   File: pipeline.py    Apache License 2.0 5 votes vote down vote up
def begin_abort(self, root_pipeline_key, abort_message):
    """Kicks off the abort process for a root pipeline and all its children.

    Args:
      root_pipeline_key: db.Key of the root pipeline to abort.
      abort_message: Message explaining why the abort happened, only saved
          into the root pipeline.

    Returns:
      True if the abort signal was sent successfully; False otherwise.
    """
    def txn():
      pipeline_record = db.get(root_pipeline_key)
      if pipeline_record is None:
        logging.warning(
            'Tried to abort root pipeline ID "%s" but it does not exist.',
            root_pipeline_key.name())
        raise db.Rollback()
      if pipeline_record.status == _PipelineRecord.ABORTED:
        logging.warning(
            'Tried to abort root pipeline ID "%s"; already in state: %s',
            root_pipeline_key.name(), pipeline_record.status)
        raise db.Rollback()
      if pipeline_record.abort_requested:
        logging.warning(
            'Tried to abort root pipeline ID "%s"; abort signal already sent.',
            root_pipeline_key.name())
        raise db.Rollback()

      pipeline_record.abort_requested = True
      pipeline_record.abort_message = abort_message
      pipeline_record.put()

      task = taskqueue.Task(
          url=self.fanout_abort_handler_path,
          params=dict(root_pipeline_key=root_pipeline_key))
      task.add(queue_name=self.queue_name, transactional=True)
      return True

    return db.run_in_transaction(txn) 
Example 59
Project: luci-py   Author: luci   File: model.py    Apache License 2.0 5 votes vote down vote up
def to_task(self):
    """Convert to a taskqueue task."""
    # Never pass params to taskqueue.Task. Use payload instead. Otherwise,
    # it's up to a particular taskqueue implementation to generate
    # payload from params. It could blow up payload size over limit.
    return taskqueue.Task(
        url=self.url,
        payload=self._payload,
        name=self.name,
        eta=self.eta,
        countdown=self.countdown,
        headers=self._headers) 
Example 60
Project: luci-py   Author: luci   File: handlers.py    Apache License 2.0 5 votes vote down vote up
def _add_task(cls,
                worker_task,
                mapreduce_spec,
                queue_name):
    """Schedule slice scanning by adding it to the task queue.

    Args:
      worker_task: a model.HugeTask task for slice. This is NOT a taskqueue
        task.
      mapreduce_spec: an instance of model.MapreduceSpec.
      queue_name: Optional queue to run on; uses the current queue of
        execution or the default queue if unspecified.
    """
    if not _run_task_hook(mapreduce_spec.get_hooks(),
                          "enqueue_worker_task",
                          worker_task,
                          queue_name):
      try:
        # Not adding transactionally because worker_task has name.
        # Named task is not allowed for transactional add.
        worker_task.add(queue_name)
      except (taskqueue.TombstonedTaskError,
              taskqueue.TaskAlreadyExistsError), e:
        logging.warning("Task %r already exists. %s: %s",
                        worker_task.name,
                        e.__class__,
                        e) 
Example 61
Project: luci-py   Author: luci   File: handlers.py    Apache License 2.0 5 votes vote down vote up
def reschedule(cls,
                 mapreduce_state,
                 mapreduce_spec,
                 serial_id,
                 queue_name=None):
    """Schedule new update status callback task.

    Args:
      mapreduce_state: mapreduce state as model.MapreduceState
      mapreduce_spec: mapreduce specification as MapreduceSpec.
      serial_id: id of the invocation as int.
      queue_name: The queue to schedule this task on. Will use the current
        queue of execution if not supplied.
    """
    task_name = ControllerCallbackHandler.get_task_name(
        mapreduce_spec, serial_id)
    task_params = ControllerCallbackHandler.controller_parameters(
        mapreduce_spec, serial_id)
    if not queue_name:
      queue_name = os.environ.get("HTTP_X_APPENGINE_QUEUENAME", "default")

    controller_callback_task = model.HugeTask(
        url=(mapreduce_spec.params["base_path"] + "/controller_callback/" +
             mapreduce_spec.mapreduce_id),
        name=task_name, params=task_params,
        countdown=parameters.config._CONTROLLER_PERIOD_SEC,
        parent=mapreduce_state,
        headers=util._get_task_headers(mapreduce_spec.mapreduce_id))

    if not _run_task_hook(mapreduce_spec.get_hooks(),
                          "enqueue_controller_task",
                          controller_callback_task,
                          queue_name):
      try:
        controller_callback_task.add(queue_name)
      except (taskqueue.TombstonedTaskError,
              taskqueue.TaskAlreadyExistsError), e:
        logging.warning("Task %r with params %r already exists. %s: %s",
                        task_name, task_params, e.__class__, e) 
Example 62
Project: luci-py   Author: luci   File: utils.py    Apache License 2.0 5 votes vote down vote up
def get_module_version_list(module_list, tainted):
  """Returns a list of pairs (module name, version name) to fetch logs for.

  Arguments:
    module_list: list of modules to list, defaults to all modules.
    tainted: if False, excludes versions with '-tainted' in their name.
  """
  result = []
  if not module_list:
    # If the function it called too often, it'll raise a OverQuotaError. So
    # cache it for 10 minutes.
    module_list = gae_memcache.get('modules_list')
    if not module_list:
      module_list = modules.get_modules()
      gae_memcache.set('modules_list', module_list, time=10*60)

  for module in module_list:
    # If the function it called too often, it'll raise a OverQuotaError.
    # Versions is a bit more tricky since we'll loose data, since versions are
    # changed much more often than modules. So cache it for 1 minute.
    key = 'modules_list-' + module
    version_list = gae_memcache.get(key)
    if not version_list:
      version_list = modules.get_versions(module)
      gae_memcache.set(key, version_list, time=60)
    result.extend(
        (module, v) for v in version_list if tainted or '-tainted' not in v)
  return result


## Task queue 
Example 63
Project: luci-py   Author: luci   File: pipeline.py    Apache License 2.0 5 votes vote down vote up
def begin_abort(self, root_pipeline_key, abort_message):
    """Kicks off the abort process for a root pipeline and all its children.

    Args:
      root_pipeline_key: db.Key of the root pipeline to abort.
      abort_message: Message explaining why the abort happened, only saved
          into the root pipeline.

    Returns:
      True if the abort signal was sent successfully; False otherwise.
    """
    def txn():
      pipeline_record = db.get(root_pipeline_key)
      if pipeline_record is None:
        logging.warning(
            'Tried to abort root pipeline ID "%s" but it does not exist.',
            root_pipeline_key.name())
        raise db.Rollback()
      if pipeline_record.status == _PipelineRecord.ABORTED:
        logging.warning(
            'Tried to abort root pipeline ID "%s"; already in state: %s',
            root_pipeline_key.name(), pipeline_record.status)
        raise db.Rollback()
      if pipeline_record.abort_requested:
        logging.warning(
            'Tried to abort root pipeline ID "%s"; abort signal already sent.',
            root_pipeline_key.name())
        raise db.Rollback()

      pipeline_record.abort_requested = True
      pipeline_record.abort_message = abort_message
      pipeline_record.put()

      task = taskqueue.Task(
          url=self.fanout_abort_handler_path,
          params=dict(root_pipeline_key=root_pipeline_key))
      task.add(queue_name=self.queue_name, transactional=True)
      return True

    return db.run_in_transaction(txn) 
Example 64
Project: luci-py   Author: luci   File: model.py    Apache License 2.0 5 votes vote down vote up
def to_task(self):
    """Convert to a taskqueue task."""
    # Never pass params to taskqueue.Task. Use payload instead. Otherwise,
    # it's up to a particular taskqueue implementation to generate
    # payload from params. It could blow up payload size over limit.
    return taskqueue.Task(
        url=self.url,
        payload=self._payload,
        name=self.name,
        eta=self.eta,
        countdown=self.countdown,
        headers=self._headers) 
Example 65
Project: luci-py   Author: luci   File: handlers.py    Apache License 2.0 5 votes vote down vote up
def _add_task(cls,
                worker_task,
                mapreduce_spec,
                queue_name):
    """Schedule slice scanning by adding it to the task queue.

    Args:
      worker_task: a model.HugeTask task for slice. This is NOT a taskqueue
        task.
      mapreduce_spec: an instance of model.MapreduceSpec.
      queue_name: Optional queue to run on; uses the current queue of
        execution or the default queue if unspecified.
    """
    if not _run_task_hook(mapreduce_spec.get_hooks(),
                          "enqueue_worker_task",
                          worker_task,
                          queue_name):
      try:
        # Not adding transactionally because worker_task has name.
        # Named task is not allowed for transactional add.
        worker_task.add(queue_name)
      except (taskqueue.TombstonedTaskError,
              taskqueue.TaskAlreadyExistsError), e:
        logging.warning("Task %r already exists. %s: %s",
                        worker_task.name,
                        e.__class__,
                        e) 
Example 66
Project: luci-py   Author: luci   File: handlers.py    Apache License 2.0 5 votes vote down vote up
def reschedule(cls,
                 mapreduce_state,
                 mapreduce_spec,
                 serial_id,
                 queue_name=None):
    """Schedule new update status callback task.

    Args:
      mapreduce_state: mapreduce state as model.MapreduceState
      mapreduce_spec: mapreduce specification as MapreduceSpec.
      serial_id: id of the invocation as int.
      queue_name: The queue to schedule this task on. Will use the current
        queue of execution if not supplied.
    """
    task_name = ControllerCallbackHandler.get_task_name(
        mapreduce_spec, serial_id)
    task_params = ControllerCallbackHandler.controller_parameters(
        mapreduce_spec, serial_id)
    if not queue_name:
      queue_name = os.environ.get("HTTP_X_APPENGINE_QUEUENAME", "default")

    controller_callback_task = model.HugeTask(
        url=(mapreduce_spec.params["base_path"] + "/controller_callback/" +
             mapreduce_spec.mapreduce_id),
        name=task_name, params=task_params,
        countdown=parameters.config._CONTROLLER_PERIOD_SEC,
        parent=mapreduce_state,
        headers=util._get_task_headers(mapreduce_spec.mapreduce_id))

    if not _run_task_hook(mapreduce_spec.get_hooks(),
                          "enqueue_controller_task",
                          controller_callback_task,
                          queue_name):
      try:
        controller_callback_task.add(queue_name)
      except (taskqueue.TombstonedTaskError,
              taskqueue.TaskAlreadyExistsError), e:
        logging.warning("Task %r with params %r already exists. %s: %s",
                        task_name, task_params, e.__class__, e) 
Example 67
Project: mlab-ns   Author: m-lab   File: handlers.py    Apache License 2.0 4 votes vote down vote up
def _schedule_slice(cls,
                      shard_state,
                      transient_shard_state,
                      queue_name=None,
                      eta=None,
                      countdown=None):
    """Schedule slice scanning by adding it to the task queue.

    Args:
      shard_state: An instance of ShardState.
      transient_shard_state: An instance of TransientShardState.
      queue_name: Optional queue to run on; uses the current queue of
        execution or the default queue if unspecified.
      eta: Absolute time when the MR should execute. May not be specified
        if 'countdown' is also supplied. This may be timezone-aware or
        timezone-naive.
      countdown: Time in seconds into the future that this MR should execute.
        Defaults to zero.
    """
    base_path = transient_shard_state.base_path
    mapreduce_spec = transient_shard_state.mapreduce_spec

    task_name = MapperWorkerCallbackHandler.get_task_name(
        transient_shard_state.shard_id,
        transient_shard_state.slice_id)
    queue_name = queue_name or os.environ.get("HTTP_X_APPENGINE_QUEUENAME",
                                              "default")

    worker_task = util.HugeTask(url=base_path + "/worker_callback",
                                params=transient_shard_state.to_dict(),
                                name=task_name,
                                eta=eta,
                                countdown=countdown)

    if not _run_task_hook(mapreduce_spec.get_hooks(),
                          "enqueue_worker_task",
                          worker_task,
                          queue_name):
      try:
        worker_task.add(queue_name, parent=shard_state)
      except (taskqueue.TombstonedTaskError,
              taskqueue.TaskAlreadyExistsError), e:
        logging.warning("Task %r with params %r already exists. %s: %s",
                        task_name,
                        transient_shard_state.to_dict(),
                        e.__class__,
                        e) 
Example 68
Project: mlab-ns   Author: m-lab   File: pipeline.py    Apache License 2.0 4 votes vote down vote up
def start(self,
            idempotence_key='',
            queue_name='default',
            base_path='/_ah/pipeline',
            return_task=False):
    """Starts a new instance of this pipeline.

    Args:
      idempotence_key: The ID to use for this Pipeline and throughout its
        asynchronous workflow to ensure the operations are idempotnent. If
        empty a starting key will be automatically assigned.
      queue_name: What queue this Pipeline's workflow should execute on.
      base_path: The relative URL path to where the Pipeline API is
        mounted for access by the taskqueue API or external requests.
      return_task: When True, a task to start this pipeline will be returned
        instead of submitted, allowing the caller to start off this pipeline
        as part of a separate transaction (potentially leaving this newly
        allocated pipeline's datastore entities in place if that separate
        transaction fails for any reason).

    Returns:
      A taskqueue.Task instance if return_task was True. This task will *not*
      have a name, thus to ensure reliable execution of your pipeline you
      should add() this task as part of a separate Datastore transaction.

    Raises:
      PipelineExistsError if the pipeline with the given idempotence key exists.
      PipelineSetupError if the pipeline could not start for any other reason.
    """
    if not idempotence_key:
      idempotence_key = uuid.uuid1().hex
    pipeline_key = db.Key.from_path(_PipelineRecord.kind(), idempotence_key)
    context = _PipelineContext('', queue_name, base_path)
    future = PipelineFuture(self.output_names, force_strict=True)
    try:
      self._set_values_internal(
          context, pipeline_key, pipeline_key, future, _PipelineRecord.WAITING)
      return context.start(self, return_task=return_task)
    except Error:
      # Pass through exceptions that originate in this module.
      raise
    except Exception, e:
      # Re-type any exceptions that were raised in dependent methods.
      raise PipelineSetupError('Error starting %s#%s: %s' % (
          self, idempotence_key, str(e))) 
Example 69
Project: mlab-ns   Author: m-lab   File: pipeline.py    Apache License 2.0 4 votes vote down vote up
def fill_slot(self, filler_pipeline_key, slot, value):
    """Fills a slot, enqueueing a task to trigger pending barriers.

    Args:
      filler_pipeline_key: db.Key or stringified key of the _PipelineRecord
        that filled this slot.
      slot: The Slot instance to fill.
      value: The serializable value to assign.

    Raises:
      UnexpectedPipelineError if the _SlotRecord for the 'slot' could not
      be found in the Datastore.
    """
    if not isinstance(filler_pipeline_key, db.Key):
      filler_pipeline_key = db.Key(filler_pipeline_key)

    if _TEST_MODE:
      slot._set_value_test(filler_pipeline_key, value)
    else:
      encoded_value = simplejson.dumps(value, sort_keys=True)
      value_text = None
      value_blob = None
      if len(encoded_value) <= _MAX_JSON_SIZE:
        value_text = db.Text(encoded_value)
      else:
        # The encoded value is too big. Save it as a blob.
        value_blob = _write_json_blob(encoded_value)

      def txn():
        slot_record = db.get(slot.key)
        if slot_record is None:
          raise UnexpectedPipelineError(
              'Tried to fill missing slot "%s" '
              'by pipeline ID "%s" with value: %r'
              % (slot.key, filler_pipeline_key.name(), value))
        # NOTE: Always take the override value here. If down-stream pipelines
        # need a consitent view of all up-stream outputs (meaning, all of the
        # outputs came from the same retry attempt of the upstream pipeline),
        # the down-stream pipeline must also wait for the 'default' output
        # of these up-stream pipelines.
        slot_record.filler = filler_pipeline_key
        slot_record.value_text = value_text
        slot_record.value_blob = value_blob
        slot_record.status = _SlotRecord.FILLED
        slot_record.fill_time = self._gettime()
        slot_record.put()
        task = taskqueue.Task(
            url=self.barrier_handler_path,
            params=dict(slot_key=slot.key),
            headers={'X-Ae-Slot-Key': slot.key,
                     'X-Ae-Filler-Pipeline-Key': filler_pipeline_key})
        task.add(queue_name=self.queue_name, transactional=True)
      db.run_in_transaction(txn)

    self.session_filled_output_names.add(slot.name) 
Example 70
Project: mlab-ns   Author: m-lab   File: pipeline.py    Apache License 2.0 4 votes vote down vote up
def transition_retry(self, pipeline_key, retry_message):
    """Marks the given pipeline as requiring another retry.

    Does nothing if all attempts have been exceeded.

    Args:
      pipeline_key: db.Key of the _PipelineRecord that needs to be retried.
      retry_message: User-supplied message indicating the reason for the retry.
    """
    def txn():
      pipeline_record = db.get(pipeline_key)
      if pipeline_record is None:
        logging.warning(
            'Tried to retry pipeline ID "%s" but it does not exist.',
            pipeline_key.name())
        raise db.Rollback()
      if pipeline_record.status not in (
             _PipelineRecord.WAITING, _PipelineRecord.RUN):
        logging.warning(
            'Tried to retry pipeline ID "%s", found bad state: %s',
            pipeline_key.name(), pipeline_record.status)
        raise db.Rollback()

      params = pipeline_record.params
      offset_seconds = (params['backoff_seconds'] *
          (params['backoff_factor'] ** pipeline_record.current_attempt))
      pipeline_record.next_retry_time = (
          self._gettime() + datetime.timedelta(seconds=offset_seconds))
      pipeline_record.current_attempt += 1
      pipeline_record.retry_message = retry_message
      pipeline_record.status = _PipelineRecord.WAITING

      if pipeline_record.current_attempt >= pipeline_record.max_attempts:
        root_pipeline_key = (
            _PipelineRecord.root_pipeline.get_value_for_datastore(
            pipeline_record))
        logging.warning(
            'Giving up on pipeline ID "%s" after %d attempt(s); causing abort '
            'all the way to the root pipeline ID "%s"', pipeline_key.name(),
            pipeline_record.current_attempt, root_pipeline_key.name())
        # NOTE: We do *not* set the status to aborted here to ensure that
        # this pipeline will be finalized before it has been marked as aborted.
        pipeline_record.abort_message = (
            'Aborting after %d attempts' % pipeline_record.current_attempt)
        task = taskqueue.Task(
            url=self.fanout_abort_handler_path,
            params=dict(root_pipeline_key=root_pipeline_key))
        task.add(queue_name=self.queue_name, transactional=True)
      else:
        task = taskqueue.Task(
            url=self.pipeline_handler_path,
            eta=pipeline_record.next_retry_time,
            params=dict(pipeline_key=pipeline_key,
                        purpose=_BarrierRecord.START,
                        attempt=pipeline_record.current_attempt),
            headers={'X-Ae-Pipeline-Key': pipeline_key})
        task.add(queue_name=self.queue_name, transactional=True)

      pipeline_record.put()

    db.run_in_transaction(txn) 
Example 71
Project: roger-api   Author: rogertalk   File: jobs.py    MIT License 4 votes vote down vote up
def update_content_request_entries():
    request_id = int(flask_extras.get_parameter('request_id'))
    wallet_id = flask_extras.get_parameter('wallet_id')
    assert wallet_id
    wallet_owner_id = int(flask_extras.get_parameter('wallet_owner_id'))
    request_key = ndb.Key('ContentRequestPublic', request_id)
    q = models.ContentRequestPublicEntry.query()
    q = q.filter(models.ContentRequestPublicEntry.request == request_key)
    q = q.filter(models.ContentRequestPublicEntry.status == 'active')
    q = q.order(-models.ContentRequestPublicEntry.created)
    futures = []
    tasks = []
    job_count = 0
    delay = 10  # 10 seconds to allow YouTube update to finish.
    content_ids = []
    for entry in q:
        task = taskqueue.Task(
            countdown=delay,
            url='/_ah/jobs/update_content_request_entry',
            params={
                'account_id': str(entry.account.id()),
                'content_id': str(entry.content.id()),
                'request_id': str(entry.request.id()),
                'wallet_id': wallet_id,
                'wallet_owner_id': str(wallet_owner_id),
                'youtube_id': entry.youtube_id,
            },
            retry_options=taskqueue.TaskRetryOptions(task_retry_limit=0))
        tasks.append(task)
        if len(tasks) == 100:
            futures.append(_add_task_list_async(tasks, queue_name=config.INTERNAL_QUEUE))
            tasks = []
        delay += 1
        job_count += 1
        content_ids.append(entry.content.id())
    del q, task, tasks
    if content_ids:
        # Update YouTube views on all content ids.
        futures.append(_update_youtube_views_async(content_ids))
        job_count += 1
    del content_ids
    _wait_all(futures)
    logging.debug('Scheduled %d update job(s) for request %d', job_count, request_id)
    return '' 
Example 72
Project: roger-api   Author: rogertalk   File: admin.py    MIT License 4 votes vote down vote up
def get_requests_request_id_identifier(request_id, identifier):
    account = models.Account.resolve(identifier)
    if not account:
        return 'Not found.', 404
    # Entry id always matches account id.
    entry_id = account.key.id()
    request_key = ndb.Key('ContentRequestPublic', request_id)
    entry_key = ndb.Key('ContentRequestPublicEntry', '%d.%d' % (request_id, entry_id))
    request, entry = ndb.get_multi([request_key, entry_key])
    if not request or not entry:
        return 'Not found.', 404
    content_id = flask_extras.get_parameter('content-id')
    if content_id:
        content_id = int(content_id)
        if not entry.content or content_id != entry.content.id():
            return 'Entry content changed since page load.', 400
    request_or_entry_changed = False
    futures = []
    # TODO: Handle more actions.
    action = flask_extras.get_parameter('action')
    if action in ('approve', 'deny'):
        if entry.status != 'pending-review':
            return 'Invalid status.', 400
        entry = models.ContentRequestPublicEntry.review(entry.key, action == 'approve')
        if entry.status == 'active':
            # Immediately update reward on entry.
            task = taskqueue.Task(
                countdown=2,
                url='/_ah/jobs/update_content_request_entry',
                params={
                    'account_id': str(entry.account.id()),
                    'content_id': str(entry.content.id()),
                    'request_id': str(entry.request.id()),
                    'wallet_id': str(request.wallet.id()),
                    'wallet_owner_id': str(request.wallet_owner.id()),
                    'youtube_id': entry.youtube_id,
                },
                retry_options=taskqueue.TaskRetryOptions(task_retry_limit=0))
            futures.append(_add_task_async(task, queue_name=config.INTERNAL_QUEUE))
        request_or_entry_changed = True
    elif action == 'reset':
        entry, _ = models.ContentRequestPublicEntry.update(entry.key, None, reset=True)
        request_or_entry_changed = True
    elif action == 'restore':
        entry = models.ContentRequestPublicEntry.restore(entry.key)
        request_or_entry_changed = True
    else:
        return 'Unknown action.', 400
    if request_or_entry_changed:
        # Let user know something happened.
        hub = notifs.Hub(entry.account)
        futures.append(hub.emit_async(notifs.ON_PUBLIC_REQUEST_UPDATE, request_id=request_id))
    ndb.Future.wait_all(futures)
    return '' 
Example 73
Project: naziscore   Author: rbanffy   File: scoring.py    GNU Affero General Public License v3.0 4 votes vote down vote up
def get_score_by_screen_name(screen_name, depth):
    # Gets the most recently updated copy, if duplicated.
    key_name = (
        '.' + screen_name if screen_name.startswith('__') else screen_name)
    try:
        score = yield ndb.Key(Score, key_name).get_async()
    except OverQuotaError:
        logging.critical('We are over quota.')
        raise ndb.Return(None)
    if score is None or (
            score.last_updated < datetime.datetime.now()
            - datetime.timedelta(days=MAX_AGE_DAYS)):
            # If we don't have one, or if we have one that's too old, we need
            # to calculate one.
        if score is not None:
            logging.info('Refreshing {}'.format(screen_name))
        else:
            logging.info('Fetching {} for the first time'.format(screen_name))
        task_name = '{}_{}'.format(
                screen_name,
                os.environ['CURRENT_VERSION_ID'].split('.')[0])
        queue_name = 'scoring-direct' if depth == 0 else 'scoring-indirect'
        try:
            _ = yield taskqueue.Task(
                name=task_name,
                params={
                    'screen_name': screen_name,
                    'depth': depth
                }).add_async(queue_name)

            # If this is a direct query, schedule an analysis of the profile
            # picture.
            if depth == 0:
                _ = yield taskqueue.Task(
                    name=task_name,
                    params={
                        'screen_name': screen_name,
                    }).add_async('profile-pic')

            # If we add it to the scoring-direct queue, we should remove
            # the corresponding task from the scoring-indirect queue at this
            # point.
            if queue_name == 'scoring-direct':
                delete_from_scoring_indirect(task_name)

        except taskqueue.TaskAlreadyExistsError:
            # We already are going to check this person. There is nothing
            # to do here.
            logging.warning(
                'Fetch for {} already scheduled on queue {}'.format(
                    task_name, queue_name))

        except taskqueue.TombstonedTaskError:
            # This task is too recent. We shouldn't try again so
            # soon. Thombstoning won't happen across different deploys, as the
            # task name has the deploy timestamp on it.
            logging.warning('Fetch for {} tombstoned'.format(task_name))
    else:
        logging.info('No need to refresh {}'.format(screen_name))

    raise ndb.Return(score) 
Example 74
Project: naziscore   Author: rbanffy   File: scoring.py    GNU Affero General Public License v3.0 4 votes vote down vote up
def get_score_by_twitter_id(twitter_id, depth):
    try:
        score = yield Score.query(Score.twitter_id == twitter_id).get_async()
    except OverQuotaError:
        logging.critical(
            'Over quota fetching {}'.format(twitter_id))
        raise ndb.Return(None)
    if score is None or (
            score.last_updated < datetime.datetime.now()
            - datetime.timedelta(days=MAX_AGE_DAYS)):
            # If we don't have one, or if we have one that's too old, we need
            # to calculate one.
        task_name = '{}_{}'.format(
                twitter_id,
                os.environ['CURRENT_VERSION_ID'].split('.')[0])
        queue_name = 'scoring-direct' if depth == 0 else 'scoring-indirect'
        try:
            _ = yield taskqueue.Task(
                name=task_name,
                params={
                    'twitter_id': twitter_id,
                    'depth': depth
                }).add_async(queue_name)

            # If this is a direct query, schedule an analysis of the profile
            # picture.
            if depth == 0:
                _ = yield taskqueue.Task(
                    name=task_name,
                    params={
                        'twitter_id': twitter_id,
                    }).add_async('profile-pic')

            # If we add it to the scoring-direct queue, we should remove
            # the corresponding task from the scoring-indirect queue at this
            # point.
            if queue_name == 'scoring-direct':
                delete_from_scoring_indirect(task_name)

        except taskqueue.TaskAlreadyExistsError:
            # We already are going to check this person. There is nothing
            # to do here.
            logging.warning(
                'Fetch for {} already scheduled on queue {}'.format(
                    task_name, queue_name))

        except taskqueue.TombstonedTaskError:
            # This task is too recent. We shouldn't try again so
            # soon. Thombstoning won't happen across different deploys, as the
            # task name has the deploy timestamp on it.
            logging.warning('Fetch for {} tombstoned'.format(task_name))
        raise ndb.Return(score)
    else:
        raise ndb.Return(score) 
Example 75
Project: server   Author: viur-framework   File: tasks.py    GNU Lesser General Public License v3.0 4 votes vote down vote up
def index(self, *args, **kwargs):
		global _callableTasks, _periodicTasks
		logging.debug("Starting maintenance-run")
		checkUpdate() #Let the update-module verify the database layout first
		logging.debug("Updatecheck complete")
		for task,intervall in _periodicTasks.items(): #Call all periodic tasks
			if intervall: #Ensure this task doesn't get called to often
				try:
					lastCall = db.Get( db.Key.from_path( "viur-task-interval", task.periodicTaskName ) )
					if lastCall["date"] > datetime.now()-timedelta( minutes=intervall ):
						logging.debug("Skipping task %s - Has already run recently." % task.periodicTaskName )
						continue
				except db.EntityNotFoundError:
					pass

			res = self.findBoundTask( task )

			if res: #Its bound, call it this way :)
				res[0]()
			else:
				task() #It seems it wasnt bound - call it as a static method

			logging.debug("Successfully called task %s" % task.periodicTaskName )
			if intervall:
				# Update its last-call timestamp
				entry = db.Entity( "viur-task-interval", name=task.periodicTaskName )
				entry["date"] = datetime.now()
				db.Put( entry )
		logging.debug("Periodic tasks complete")
		for currentTask in db.Query("viur-queued-tasks").iter(): #Look for queued tasks
			db.Delete( currentTask.key() )
			if currentTask["taskid"] in _callableTasks:
				task = _callableTasks[ currentTask["taskid"] ]()
				tmpDict = {}
				for k in currentTask.keys():
					if k == "taskid":
						continue
					tmpDict[ k ] = json.loads( currentTask[ k ] )
				try:
					task.execute( **tmpDict )
				except Exception as e:
					logging.error("Error executing Task")
					logging.exception( e )
		logging.debug("Scheduled tasks complete") 
Example 76
Project: server   Author: viur-framework   File: tasks.py    GNU Lesser General Public License v3.0 4 votes vote down vote up
def callDeferred( func ):
	"""
		This is a decorator, which allways calls the function deferred.
		Unlike Googles implementation, this one works (with bound functions)
	"""
	if "viur_doc_build" in dir(sys):
		return(func)
	__undefinedFlag_ = object()
	def mkDefered( func, self=__undefinedFlag_, *args,  **kwargs ):
		from server.utils import getCurrentUser
		try:
			req = request.current.get()
		except: #This will fail for warmup requests
			req = None
		if req is not None and "HTTP_X_APPENGINE_TASKRETRYCOUNT".lower() in [x.lower() for x in os.environ.keys()] and not "DEFERED_TASK_CALLED" in dir( req ): #This is the deferred call
			req.DEFERED_TASK_CALLED = True #Defer recursive calls to an deferred function again.
			if self is __undefinedFlag_:
				return func(*args, **kwargs)
			else:
				return func(self, *args, **kwargs)
		else:
			try:
				funcPath = "%s/%s" % (self.modulePath, func.func_name )
				command = "rel"
			except:
				funcPath = "%s.%s" % ( func.__name__, func.__module__ )
				if self!=__undefinedFlag_:
					args = (self,)+args #Reappend self to args, as this function is (hopefully) unbound
				command = "unb"
			taskargs = dict((x, kwargs.pop(("_%s" % x), None))  for x in ("countdown", "eta", "name", "target", "retry_options"))
			taskargs["url"] = "/_tasks/deferred"
			transactional = kwargs.pop("_transactional", False)
			taskargs["headers"] = {"Content-Type": "application/octet-stream"}
			queue = kwargs.pop("_queue", "default")
			# Try to preserve the important data from the current environment
			env = {"user": None}
			usr = getCurrentUser()
			if usr:
				env["user"] = {"key": usr["key"],
				               "name": usr["name"],
				               "access": usr["access"]}
			try:
				env["lang"] = request.current.get().language
			except AttributeError: #This isn't originating from a normal request
				pass
			if conf["viur.tasks.customEnvironmentHandler"]:
				# Check if this project relies on additional environmental variables and serialize them too
				assert isinstance(conf["viur.tasks.customEnvironmentHandler"], tuple) \
					and len(conf["viur.tasks.customEnvironmentHandler"])==2 \
					and callable(conf["viur.tasks.customEnvironmentHandler"][0]), \
					"Your customEnvironmentHandler must be a tuple of two callable if set!"
				env["custom"] = conf["viur.tasks.customEnvironmentHandler"][0]()
			pickled = json.dumps((command, (funcPath, args, kwargs, env)))
			task = taskqueue.Task(payload=pickled, **taskargs)
			return task.add(queue, transactional=transactional)
	global _deferedTasks
	_deferedTasks[ "%s.%s" % ( func.__name__, func.__module__ ) ] = func
	return( lambda *args, **kwargs: mkDefered( func, *args, **kwargs) ) 
Example 77
Project: luci-py   Author: luci   File: pipeline.py    Apache License 2.0 4 votes vote down vote up
def start(self,
            idempotence_key='',
            queue_name='default',
            base_path='/_ah/pipeline',
            return_task=False,
            countdown=None,
            eta=None):
    """Starts a new instance of this pipeline.

    Args:
      idempotence_key: The ID to use for this Pipeline and throughout its
        asynchronous workflow to ensure the operations are idempotent. If
        empty a starting key will be automatically assigned.
      queue_name: What queue this Pipeline's workflow should execute on.
      base_path: The relative URL path to where the Pipeline API is
        mounted for access by the taskqueue API or external requests.
      return_task: When True, a task to start this pipeline will be returned
        instead of submitted, allowing the caller to start off this pipeline
        as part of a separate transaction (potentially leaving this newly
        allocated pipeline's datastore entities in place if that separate
        transaction fails for any reason).
      countdown: Time in seconds into the future that this Task should execute.
        Defaults to zero.
      eta: A datetime.datetime specifying the absolute time at which the task
        should be executed. Must not be specified if 'countdown' is specified.
        This may be timezone-aware or timezone-naive. If None, defaults to now.
        For pull tasks, no worker will be able to lease this task before the
        time indicated by eta.

    Returns:
      A taskqueue.Task instance if return_task was True. This task will *not*
      have a name, thus to ensure reliable execution of your pipeline you
      should add() this task as part of a separate Datastore transaction.

    Raises:
      PipelineExistsError if the pipeline with the given idempotence key exists.
      PipelineSetupError if the pipeline could not start for any other reason.
    """
    if not idempotence_key:
      idempotence_key = uuid.uuid4().hex
    elif not isinstance(idempotence_key, unicode):
      try:
        idempotence_key.encode('utf-8')
      except UnicodeDecodeError:
        idempotence_key = hashlib.sha1(idempotence_key).hexdigest()

    pipeline_key = db.Key.from_path(_PipelineRecord.kind(), idempotence_key)
    context = _PipelineContext('', queue_name, base_path)
    future = PipelineFuture(self.output_names, force_strict=True)
    try:
      self._set_values_internal(
          context, pipeline_key, pipeline_key, future, _PipelineRecord.WAITING)
      return context.start(
          self, return_task=return_task, countdown=countdown, eta=eta)
    except Error:
      # Pass through exceptions that originate in this module.
      raise
    except Exception, e:
      # Re-type any exceptions that were raised in dependent methods.
      raise PipelineSetupError('Error starting %s#%s: %s' % (
          self, idempotence_key, str(e))) 
Example 78
Project: luci-py   Author: luci   File: pipeline.py    Apache License 2.0 4 votes vote down vote up
def fill_slot(self, filler_pipeline_key, slot, value):
    """Fills a slot, enqueueing a task to trigger pending barriers.

    Args:
      filler_pipeline_key: db.Key or stringified key of the _PipelineRecord
        that filled this slot.
      slot: The Slot instance to fill.
      value: The serializable value to assign.

    Raises:
      UnexpectedPipelineError if the _SlotRecord for the 'slot' could not
      be found in the Datastore.
    """
    if not isinstance(filler_pipeline_key, db.Key):
      filler_pipeline_key = db.Key(filler_pipeline_key)

    if _TEST_MODE:
      slot._set_value_test(filler_pipeline_key, value)
    else:
      encoded_value = json.dumps(value,
                                       sort_keys=True,
                                       cls=mr_util.JsonEncoder)
      value_text = None
      value_blob = None
      if len(encoded_value) <= _MAX_JSON_SIZE:
        value_text = db.Text(encoded_value)
      else:
        # The encoded value is too big. Save it as a blob.
        value_blob = _write_json_blob(encoded_value, filler_pipeline_key.name())

      def txn():
        slot_record = db.get(slot.key)
        if slot_record is None:
          raise UnexpectedPipelineError(
              'Tried to fill missing slot "%s" '
              'by pipeline ID "%s" with value: %r'
              % (slot.key, filler_pipeline_key.name(), value))
        # NOTE: Always take the override value here. If down-stream pipelines
        # need a consitent view of all up-stream outputs (meaning, all of the
        # outputs came from the same retry attempt of the upstream pipeline),
        # the down-stream pipeline must also wait for the 'default' output
        # of these up-stream pipelines.
        slot_record.filler = filler_pipeline_key
        slot_record.value_text = value_text
        slot_record.value_blob = value_blob
        slot_record.status = _SlotRecord.FILLED
        slot_record.fill_time = self._gettime()
        slot_record.put()
        task = taskqueue.Task(
            url=self.barrier_handler_path,
            params=dict(
                slot_key=slot.key,
                use_barrier_indexes=True),
            headers={'X-Ae-Slot-Key': slot.key,
                     'X-Ae-Filler-Pipeline-Key': filler_pipeline_key})
        task.add(queue_name=self.queue_name, transactional=True)
      db.run_in_transaction_options(
          db.create_transaction_options(propagation=db.ALLOWED), txn)

    self.session_filled_output_names.add(slot.name) 
Example 79
Project: luci-py   Author: luci   File: pipeline.py    Apache License 2.0 4 votes vote down vote up
def fill_slot(self, filler_pipeline_key, slot, value):
    """Fills a slot, enqueueing a task to trigger pending barriers.

    Args:
      filler_pipeline_key: db.Key or stringified key of the _PipelineRecord
        that filled this slot.
      slot: The Slot instance to fill.
      value: The serializable value to assign.

    Raises:
      UnexpectedPipelineError if the _SlotRecord for the 'slot' could not
      be found in the Datastore.
    """
    if not isinstance(filler_pipeline_key, db.Key):
      filler_pipeline_key = db.Key(filler_pipeline_key)

    if _TEST_MODE:
      slot._set_value_test(filler_pipeline_key, value)
    else:
      encoded_value = json.dumps(value,
                                       sort_keys=True,
                                       cls=mr_util.JsonEncoder)
      value_text = None
      value_blob = None
      if len(encoded_value) <= _MAX_JSON_SIZE:
        value_text = db.Text(encoded_value)
      else:
        # The encoded value is too big. Save it as a blob.
        value_blob = _write_json_blob(encoded_value, filler_pipeline_key.name())

      def txn():
        slot_record = db.get(slot.key)
        if slot_record is None:
          raise UnexpectedPipelineError(
              'Tried to fill missing slot "%s" '
              'by pipeline ID "%s" with value: %r'
              % (slot.key, filler_pipeline_key.name(), value))
        # NOTE: Always take the override value here. If down-stream pipelines
        # need a consitent view of all up-stream outputs (meaning, all of the
        # outputs came from the same retry attempt of the upstream pipeline),
        # the down-stream pipeline must also wait for the 'default' output
        # of these up-stream pipelines.
        slot_record.filler = filler_pipeline_key
        slot_record.value_text = value_text
        slot_record.value_blob = value_blob
        slot_record.status = _SlotRecord.FILLED
        slot_record.fill_time = self._gettime()
        slot_record.put()
        task = taskqueue.Task(
            url=self.barrier_handler_path,
            params=dict(
                slot_key=slot.key,
                use_barrier_indexes=True),
            headers={'X-Ae-Slot-Key': slot.key,
                     'X-Ae-Filler-Pipeline-Key': filler_pipeline_key})
        task.add(queue_name=self.queue_name, transactional=True)
      db.run_in_transaction_options(
          db.create_transaction_options(propagation=db.ALLOWED), txn)

    self.session_filled_output_names.add(slot.name) 
Example 80
Project: luci-py   Author: luci   File: pipeline.py    Apache License 2.0 4 votes vote down vote up
def post(self):
    if 'HTTP_X_APPENGINE_TASKNAME' not in self.request.environ:
      self.response.set_status(403)
      return

    context = _PipelineContext.from_environ(self.request.environ)

    # Set of stringified db.Keys of children to run.
    all_pipeline_keys = set()

    # For backwards compatibility with the old style of fan-out requests.
    all_pipeline_keys.update(self.request.get_all('pipeline_key'))

    # Fetch the child pipelines from the parent. This works around the 10KB
    # task payload limit. This get() is consistent-on-read and the fan-out
    # task is enqueued in the transaction that updates the parent, so the
    # fanned_out property is consistent here.
    parent_key = self.request.get('parent_key')
    child_indexes = [int(x) for x in self.request.get_all('child_indexes')]
    if parent_key:
      parent_key = db.Key(parent_key)
      parent = db.get(parent_key)
      for index in child_indexes:
        all_pipeline_keys.add(str(parent.fanned_out[index]))

    all_tasks = []
    all_pipelines = db.get([db.Key(pipeline_key) for pipeline_key in all_pipeline_keys])
    for child_pipeline in all_pipelines:
      if child_pipeline is None:
        continue
      pipeline_key = str(child_pipeline.key())
      all_tasks.append(taskqueue.Task(
          url=context.pipeline_handler_path,
          params=dict(pipeline_key=pipeline_key),
          target=child_pipeline.params.get('target'),
          headers={'X-Ae-Pipeline-Key': pipeline_key},
          name='ae-pipeline-fan-out-' + child_pipeline.key().name()))

    batch_size = 100  # Limit of taskqueue API bulk add.
    for i in xrange(0, len(all_tasks), batch_size):
      batch = all_tasks[i:i+batch_size]
      try:
        taskqueue.Queue(context.queue_name).add(batch)
      except (taskqueue.TombstonedTaskError, taskqueue.TaskAlreadyExistsError):
        pass