Python google.appengine.ext.db.Key() Examples

def start_test(self, idempotence_key=None, base_path='', **kwargs):
    """Starts this pipeline in test fashion.

    Args:
      idempotence_key: Dummy idempotence_key to use for this root pipeline.
      base_path: Dummy base URL path to use for this root pipeline.
      kwargs: Ignored keyword arguments usually passed to start().
    """
    if not idempotence_key:
      idempotence_key = uuid.uuid1().hex
    pipeline_key = db.Key.from_path(_PipelineRecord.kind(), idempotence_key)
    context = _PipelineContext('', 'default', base_path)
    future = PipelineFuture(self.output_names, force_strict=True)
    self._set_values_internal(
        context, pipeline_key, pipeline_key, future, _PipelineRecord.WAITING)
    context.start_test(self)

  # Pipeline control methods. 

def run_count(make_query, update_counter, counter):
    """Scans the entities matching a query up to FETCH_LIMIT.
    
    Returns False if we finished counting all entries."""
    # Get the next batch of entities.
    query = make_query()
    if counter.last_key:
        query = query.filter('__key__ >', db.Key(counter.last_key))
    entities = query.order('__key__').fetch(FETCH_LIMIT)
    if not entities:
        counter.last_key = ''
        return False

    # Pass the entities to the counting function.
    for entity in entities:
        update_counter(counter, entity)

    # Remember where we left off.
    counter.last_key = str(entities[-1].key())
    return True 

def _get_unapplied_jobs_accross_namespaces(self,
                                             namespace_start,
                                             namespace_end,
                                             app):
    filters = {"__key__ >=": db.Key.from_path("__namespace__",
                                              namespace_start or 1,
                                              _app=app),
               "__key__ <=": db.Key.from_path("__namespace__",
                                              namespace_end or 1,
                                              _app=app),
               self.UNAPPLIED_LOG_FILTER: self.start_time_us}
    unapplied_query = datastore.Query(filters=filters, keys_only=True, _app=app)
    return unapplied_query.Get(
        limit=self._batch_size,
        config=datastore_rpc.Configuration(
            deadline=self.UNAPPLIED_QUERY_DEADLINE)) 

def FixFirefoxBeta(request):
  query = db.Query(models.user_agent.UserAgent)
  query.filter('family =', 'Firefox')
  query.filter('v3 =', 'b5')
  key = request.GET.get('key')
  if key:
    query.filter('__key__ >', db.Key(key))
  query.order('__key__')
  user_agent = query.get()
  if not user_agent:
    return http.HttpResponse('All Done!')

  # Do something with user_agent here.
  user_agent.family = 'Firefox Beta'
  user_agent.save()

  params = {
    'next_url': '/admin/ua/ffbeta?key=%s' % user_agent.key(),
    'current_name': user_agent.get_string_list(),
    'next_name': 'nextosity'
  }
  return util.Render(request, 'update_datastore.html', params) 

def ancestor(self, ancestor):
        """Sets an ancestor for this query.

        This restricts the query to only return results that descend from
        a given model instance. In other words, all of the results will
        have the ancestor as their parent, or parent's parent, etc.  The
        ancestor itself is also a possible result!

        Args:
          ancestor: Model or Key (that has already been saved)

        Returns:
          Self to support method chaining.
        """
        self._ancestor = ancestor
        return self 

def cleanup(self):
    """Clean up this Pipeline and all Datastore records used for coordination.

    Only works when called on a root pipeline. Child pipelines will ignore
    calls to this method.

    After this method is called, Pipeline.from_id() and related status
    methods will return inconsistent or missing results. This method is
    fire-and-forget and asynchronous.
    """
    if self._root_pipeline_key is None:
      raise UnexpectedPipelineError(
          'Could not cleanup Pipeline with unknown root pipeline ID.')
    if not self.is_root:
      return
    task = taskqueue.Task(
        params=dict(root_pipeline_key=self._root_pipeline_key),
        url=self.base_path + '/cleanup',
        headers={'X-Ae-Pipeline-Key': self._root_pipeline_key})
    taskqueue.Queue(self.queue_name).add(task) 

def _set_values_internal(self,
                           context,
                           pipeline_key,
                           root_pipeline_key,
                           outputs,
                           result_status):
    """Sets the user-visible values provided as an API by this class.

    Args:
      context: The _PipelineContext used for this Pipeline.
      pipeline_key: The db.Key of this pipeline.
      root_pipeline_key: The db.Key of the root pipeline.
      outputs: The PipelineFuture for this pipeline.
      result_status: The result status of this pipeline.
    """
    self._context = context
    self._pipeline_key = pipeline_key
    self._root_pipeline_key = root_pipeline_key
    self._result_status = result_status
    self.outputs = outputs 

def _key_for_namespace(namespace, app):
  """Return the __namespace__ key for a namespace.

  Args:
    namespace: The namespace whose key is requested.
    app: The id of the application that the key belongs to.

  Returns:
    A db.Key representing the namespace.
  """
  if namespace:
    return db.Key.from_path(metadata.Namespace.KIND_NAME,
                            namespace,
                            _app=app)
  else:
    return db.Key.from_path(metadata.Namespace.KIND_NAME,
                            metadata.Namespace.EMPTY_NAMESPACE_ID,
                            _app=app) 

def _key_for_namespace(namespace, app):
  """Return the __namespace__ key for a namespace.

  Args:
    namespace: The namespace whose key is requested.
    app: The id of the application that the key belongs to.

  Returns:
    A db.Key representing the namespace.
  """
  if namespace:
    return db.Key.from_path(metadata.Namespace.KIND_NAME,
                            namespace,
                            _app=app)
  else:
    return db.Key.from_path(metadata.Namespace.KIND_NAME,
                            metadata.Namespace.EMPTY_NAMESPACE_ID,
                            _app=app) 

def to_barrier_key(cls, barrier_index_key):
    """Converts a _BarrierIndex key to a _BarrierRecord key.

    Args:
      barrier_index_key: db.Key for a _BarrierIndex entity.

    Returns:
      db.Key for the corresponding _BarrierRecord entity.
    """
    barrier_index_path = barrier_index_key.to_path()

    # Pick out the items from the _BarrierIndex key path that we need to
    # construct the _BarrierRecord key path.
    (pipeline_kind, dependent_pipeline_id,
     unused_kind, purpose) = barrier_index_path[-4:]

    barrier_record_path = (
        pipeline_kind, dependent_pipeline_id,
        _BarrierRecord.kind(), purpose)

    return db.Key.from_path(*barrier_record_path) 

def start_test(self, idempotence_key=None, base_path='', **kwargs):
    """Starts this pipeline in test fashion.

    Args:
      idempotence_key: Dummy idempotence_key to use for this root pipeline.
      base_path: Dummy base URL path to use for this root pipeline.
      kwargs: Ignored keyword arguments usually passed to start().
    """
    if not idempotence_key:
      idempotence_key = uuid.uuid4().hex
    pipeline_key = db.Key.from_path(_PipelineRecord.kind(), idempotence_key)
    context = _PipelineContext('', 'default', base_path)
    future = PipelineFuture(self.output_names, force_strict=True)
    self._set_values_internal(
        context, pipeline_key, pipeline_key, future, _PipelineRecord.WAITING)
    context.start_test(self)

  # Pipeline control methods. 

def cleanup(self):
    """Clean up this Pipeline and all Datastore records used for coordination.

    Only works when called on a root pipeline. Child pipelines will ignore
    calls to this method.

    After this method is called, Pipeline.from_id() and related status
    methods will return inconsistent or missing results. This method is
    fire-and-forget and asynchronous.
    """
    if self._root_pipeline_key is None:
      raise UnexpectedPipelineError(
          'Could not cleanup Pipeline with unknown root pipeline ID.')
    if not self.is_root:
      return
    task = taskqueue.Task(
        params=dict(root_pipeline_key=self._root_pipeline_key),
        url=self.base_path + '/cleanup',
        headers={'X-Ae-Pipeline-Key': self._root_pipeline_key})
    taskqueue.Queue(self.queue_name).add(task) 

def _set_values_internal(self,
                           context,
                           pipeline_key,
                           root_pipeline_key,
                           outputs,
                           result_status):
    """Sets the user-visible values provided as an API by this class.

    Args:
      context: The _PipelineContext used for this Pipeline.
      pipeline_key: The db.Key of this pipeline.
      root_pipeline_key: The db.Key of the root pipeline.
      outputs: The PipelineFuture for this pipeline.
      result_status: The result status of this pipeline.
    """
    self._context = context
    self._pipeline_key = pipeline_key
    self._root_pipeline_key = root_pipeline_key
    self._result_status = result_status
    self.outputs = outputs 

def __get_value(self, name):
    """Gets a `BlobInfo` value, loading an entity if necessary.

    This method allows lazy loading of the underlying datastore entity.  It
    should never be invoked directly.

    Args:
      name: The name of property for which you want to retrieve the value.

    Returns:
      The value of the `BlobInfo` property from the entity.
    """
    if self.__entity is None:
      self.__entity = datastore.Get(
          datastore_types.Key.from_path(
              self.kind(), str(self.__key), namespace=''))
    return self.__entity.get(name, None) 

def _key_for_namespace(namespace, app):
  """Return the __namespace__ key for a namespace.

  Args:
    namespace: The namespace whose key is requested.
    app: The id of the application that the key belongs to.

  Returns:
    A db.Key representing the namespace.
  """
  if namespace:
    return db.Key.from_path(metadata.Namespace.KIND_NAME,
                            namespace,
                            _app=app)
  else:
    return db.Key.from_path(metadata.Namespace.KIND_NAME,
                            metadata.Namespace.EMPTY_NAMESPACE_ID,
                            _app=app) 

def run(self, mr_type, encoded_key, output):
    logging.debug("output is %s" % str(output))
    key = db.Key(encoded=encoded_key)
    m = FileMetadata.get(key)

    blobstore_filename = "/gs" + output[0]
    blobstore_gs_key = blobstore.create_gs_key(blobstore_filename)
    url_path = "/blobstore/" + blobstore_gs_key

    if mr_type == "WordCount":
      m.wordcount_link = url_path
    elif mr_type == "Index":
      m.index_link = url_path
    elif mr_type == "Phrases":
      m.phrases_link = url_path

    m.put() 

def _key_for_namespace(namespace, app):
  """Return the __namespace__ key for a namespace.

  Args:
    namespace: The namespace whose key is requested.
    app: The id of the application that the key belongs to.

  Returns:
    A db.Key representing the namespace.
  """
  if namespace:
    return db.Key.from_path(metadata.Namespace.KIND_NAME,
                            namespace,
                            _app=app)
  else:
    return db.Key.from_path(metadata.Namespace.KIND_NAME,
                            metadata.Namespace.EMPTY_NAMESPACE_ID,
                            _app=app) 

def run(self, mr_type, encoded_key, output):
        logging.debug("output is %s" % str(output))
        key = db.Key(encoded=encoded_key)
        m = FileMetadata.get(key)

        if mr_type == "WordCount":
            m.wordcount_link = output[0]
        elif mr_type == "Index":
            m.index_link = output[0]
        elif mr_type == "Phrases":
            m.phrases_link = output[0]

        m.put() 

def __normalize_and_convert_keys(cls, keys):
    """Normalizes and converts all keys to `BlobKey` type.

    This method is based on `datastore.NormalizeAndTypeCheck()`.

    Args:
      keys: A single key or a list or tuple of keys. Keys can be a string or
          a `BlobKey`.

    Returns:
      A single key or a list with all of the strings replaced by `BlobKey`
      instances.
    """
    if isinstance(keys, (list, tuple)):
      multiple = True

      keys = list(keys)
    else:
      multiple = False
      keys = [keys]

    for index, key in enumerate(keys):
      if not isinstance(key, (basestring, BlobKey)):
        raise datastore_errors.BadArgumentError(
            'Expected string or BlobKey; received %s (a %s)' % (
                key,
                datastore.typename(key)))
      keys[index] = datastore.Key.from_path(cls.kind(), str(key), namespace='')

    if multiple:
      return keys
    else:
      return keys[0] 

def transition_complete(self, pipeline_key):
    """Marks the given pipeline as complete.

    Does nothing if the pipeline is no longer in a state that can be completed.

    Args:
      pipeline_key: db.Key of the _PipelineRecord that has completed.
    """
    def txn():
      pipeline_record = db.get(pipeline_key)
      if pipeline_record is None:
        logging.warning(
            'Tried to mark pipeline ID "%s" as complete but it does not exist.',
            pipeline_key.name())
        raise db.Rollback()
      if pipeline_record.status not in (
          _PipelineRecord.WAITING, _PipelineRecord.RUN):
        logging.warning(
            'Tried to mark pipeline ID "%s" as complete, found bad state: %s',
            pipeline_key.name(), pipeline_record.status)
        raise db.Rollback()

      pipeline_record.status = _PipelineRecord.DONE
      pipeline_record.finalized_time = self._gettime()
      pipeline_record.put()

    db.run_in_transaction(txn) 

def SubmitChanges(request):
  logging.info('^^^^^^^^^^^^^^^^^^^^ SubmitChanges')
  encoded_ua_keys = [key[3:] for key in request.GET if key.startswith('ac_')]
  logging.info('Encoded ua keys: %s' % ', '.join(encoded_ua_keys))
  update_user_agents = []
  for encoded_key in encoded_ua_keys:
    action = request.REQUEST['ac_%s' % encoded_key]
    ua = db.get(db.Key(encoded=encoded_key))
    if action == 'confirm' and not ua.confirmed:
      ua.confirmed = True
      update_user_agents.append(ua)
    if action == 'unconfirm' and ua.confirmed:
      ua.confirmed = False
      update_user_agents.append(ua)
    if action == 'change' and not ua.changed:
      change_to = request.REQUEST['cht_%s' % key]
      if change_to != ua.pretty():
        #UserAgent.parse_to_string_list(change_to)
        #ua.family =
        pass
  logging.info('Update user agents: %s' % ', '.join([ua.string for ua in update_user_agents]))
  db.put(update_user_agents)
  return http.HttpResponseRedirect(
      '/admin/confirm-ua?browser=%s&confirmed=%s&cursor=%s' %
      (request.REQUEST.get('browser', ''),
       request.REQUEST.get('confirmed', ''),
       request.REQUEST.get('cursor', ''))) 

def __cmp__(self, other):
    """Compare two key ranges.

    Key ranges with a value of None for key_start or key_end, are always
    considered to have include_start=False or include_end=False, respectively,
    when comparing.  Since None indicates an unbounded side of the range,
    the include specifier is meaningless.  The ordering generated is total
    but somewhat arbitrary.

    Args:
      other: An object to compare to this one.

    Returns:
      -1: if this key range is less than other.
      0:  if this key range is equal to other.
      1: if this key range is greater than other.
    """
    if not isinstance(other, KeyRange):
      return 1

    self_list = [self.key_start, self.key_end, self.direction,
                 self.include_start, self.include_end, self._app,
                 self.namespace]
    if not self.key_start:
      self_list[3] = False
    if not self.key_end:
      self_list[4] = False

    other_list = [other.key_start,
                  other.key_end,
                  other.direction,
                  other.include_start,
                  other.include_end,
                  other._app,
                  other.namespace]
    if not other.key_start:
      other_list[3] = False
    if not other.key_end:
      other_list[4] = False

    return cmp(self_list, other_list) 

def begin_abort(self, root_pipeline_key, abort_message):
    """Kicks off the abort process for a root pipeline and all its children.

    Args:
      root_pipeline_key: db.Key of the root pipeline to abort.
      abort_message: Message explaining why the abort happened, only saved
          into the root pipeline.

    Returns:
      True if the abort signal was sent successfully; False otherwise.
    """
    def txn():
      pipeline_record = db.get(root_pipeline_key)
      if pipeline_record is None:
        logging.warning(
            'Tried to abort root pipeline ID "%s" but it does not exist.',
            root_pipeline_key.name())
        raise db.Rollback()
      if pipeline_record.status == _PipelineRecord.ABORTED:
        logging.warning(
            'Tried to abort root pipeline ID "%s"; already in state: %s',
            root_pipeline_key.name(), pipeline_record.status)
        raise db.Rollback()
      if pipeline_record.abort_requested:
        logging.warning(
            'Tried to abort root pipeline ID "%s"; abort signal already sent.',
            root_pipeline_key.name())
        raise db.Rollback()

      pipeline_record.abort_requested = True
      pipeline_record.abort_message = abort_message
      pipeline_record.put()

      task = taskqueue.Task(
          url=self.fanout_abort_handler_path,
          params=dict(root_pipeline_key=root_pipeline_key))
      task.add(queue_name=self.queue_name, transactional=True)
      return True

    return db.run_in_transaction(txn) 

def from_json(json_str):
    """Deserialize KeyRange from its json representation.

    Args:
      json_str: string with json representation created by key_range_to_json.

    Returns:
      deserialized KeyRange instance.
    """
    if simplejson is None:
      raise SimplejsonUnavailableError(
          "JSON functionality requires json or simplejson to be available")

    def key_from_str(key_str):
      if key_str:
        return db.Key(key_str)
      else:
        return None

    json = simplejson.loads(json_str)
    return KeyRange(key_from_str(json["key_start"]),
                    key_from_str(json["key_end"]),
                    json["direction"],
                    json["include_start"],
                    json["include_end"],
                    json.get("namespace"),
                    _app=json.get("_app")) 

def getFirstKeyForUser(username):
    """Helper function that returns the first possible key a user could own.

    This is useful for table scanning, in conjunction with getLastKeyForUser.

    Args:
      username: The given user's e-mail address.
    Returns:
      The internal key representing the earliest possible key that a user could
      own (although the value of this key is not able to be used for actual
      user data).
    """

    return db.Key.from_path("FileMetadata", username + FileMetadata.__SEP) 

def UpdateCategory(request):
  category = request.REQUEST.get('category')
  user_agent_key = request.REQUEST.get('user_agent_key')
  if not category:
    logging.info('cron.UserAgentGroup: No category')
    return http.HttpResponse('No category')

  test_set = all_test_sets.GetTestSet(category)
  if not test_set:
    logging.info('cron.UserAgentGroup: Bad category: %s', category)
    return http.HttpResponse('Bad category: %s' % category)

  if not user_agent_key:
    logging.info('cron.UserAgentGroup: No key')
    return http.HttpResponse('No key')
  try:
    user_agent = UserAgent.get(db.Key(user_agent_key))
  except db.BadKeyError:
    logging.info('cron.UserAgentGroup: Invalid UserAgent key: %s', user_agent_key)
    return http.HttpResponse('Invalid UserAgent key: %s' % user_agent_key)
  if user_agent:
    logging.info('cron.UserAgentGroup: UpdateCategory(%s, %s)', category, user_agent_key)
    try:
      result_stats.UpdateCategory(category, user_agent)
      return http.HttpResponse('Done with UserAgent key=%s' % user_agent_key)
    except:
      return http.HttpResponse('Got a BadValueError. eff this one.')
  else:
    return http.HttpResponse('No user_agent with this key.') 

def handle(self):
    cursor = self.request.get("cursor")
    count = int(self.request.get("count", "50"))

    query = model.MapreduceState.all()
    if cursor:
      query.filter("__key__ >=", db.Key(cursor))
    query.order("__key__")

    jobs_list = query.fetch(count + 1)
    if len(jobs_list) == (count + 1):
      self.json_response["cursor"] = str(jobs_list[-1].key())
      jobs_list = jobs_list[:-1]

    all_jobs = []
    for job in jobs_list:
      out = {
          # Data shared between overview and detail pages.
          "name": job.mapreduce_spec.name,
          "mapreduce_id": job.mapreduce_spec.mapreduce_id,
          "active": job.active,
          "start_timestamp_ms":
              int(time.mktime(job.start_time.utctimetuple()) * 1000),
          "updated_timestamp_ms":
              int(time.mktime(job.last_poll_time.utctimetuple()) * 1000),

          # Specific to overview page.
          "chart_url": job.sparkline_url,
          "chart_width": job.chart_width,
          "active_shards": job.active_shards,
          "shards": job.mapreduce_spec.mapper.shard_count,
      }
      if job.result_status:
        out["result_status"] = job.result_status
      all_jobs.append(out)

    self.json_response["jobs"] = all_jobs 

def _iter_key_range(self, k_range):
    """Yields a db.Key and the value that should be yielded by self.__iter__().

    Args:
      k_range: The key_range.KeyRange to iterate over.

    Yields:
      A 2-tuple containing the last db.Key processed and the value that should
      be yielded by __iter__. The returned db.Key will be used to determine the
      InputReader's current position in self._current_key_range.
    """
    raise NotImplementedError("_iter_key_range() not implemented in %s" %
                              self.__class__) 

def __init__(self, *args, **kwargs):
    """Initializer.

    Args:
      *args: The positional arguments for this function-object.
      **kwargs: The keyword arguments for this function-object.
    """
    self.args = args
    self.kwargs = kwargs
    self.outputs = None
    self.backoff_seconds = _DEFAULT_BACKOFF_SECONDS
    self.backoff_factor = _DEFAULT_BACKOFF_FACTOR
    self.max_attempts = _DEFAULT_MAX_ATTEMPTS
    self.target = None
    self.task_retry = False
    self._current_attempt = 0
    self._root_pipeline_key = None
    self._pipeline_key = None
    self._context = None
    self._result_status = None
    self._set_class_path()
    # Introspectively set the target so pipelines stick to the version it
    # started.
    self.target = mr_util._get_task_target()

    if _TEST_MODE:
      self._context = _PipelineContext('', 'default', '')
      self._root_pipeline_key = _TEST_ROOT_PIPELINE_KEY
      self._pipeline_key = db.Key.from_path(
          _PipelineRecord.kind(), uuid.uuid4().hex)
      self.outputs = PipelineFuture(self.output_names)
      self._context.evaluate_test(self) 

def getLastKeyForUser(username):
    """Helper function that returns the last possible key a user could own.

    This is useful for table scanning, in conjunction with getFirstKeyForUser.

    Args:
      username: The given user's e-mail address.
    Returns:
      The internal key representing the last possible key that a user could
      own (although the value of this key is not able to be used for actual
      user data).
    """

    return db.Key.from_path("FileMetadata", username + FileMetadata.__NEXT)