Python google.appengine.ext.ndb.Query() Examples

def make_query(self, ns):
    """Make a query of entities within this range.

    Query options are not supported. They should be specified when the query
    is run.

    Args:
      ns: namespace of this query.

    Returns:
      a db.Query or ndb.Query, depends on the model class's type.
    """
    if issubclass(self.model_class, db.Model):
      query = db.Query(self.model_class, namespace=ns)
      for f in self.filters:
        query.filter("%s %s" % (f[0], f[1]), f[2])
    else:
      query = self.model_class.query(namespace=ns)
      for f in self.filters:
        query = query.filter(ndb.FilterNode(*f))
    return query 

def respond_with_query_page(self, query, callback=None):
    """Sets the handler response to a page of query results.

    Args:
      query: ndb.Query or None, A query object which will be executed and the
          results used as the page contents. If None, the response will simulate
          the results of an empty query.
      callback: function, A function that will be applied to the list of query
          results.
    """
    # NOTE: AFAICT, ndb has no notion of a "Null Query" so we need an
    # extra code path for when we don't want to return any results.
    if not query:
      logging.info('No query results are being returned')
      self.respond_with_page([], None, False)
      return

    content, cursor, more = query.fetch_page(
        self.per_page, start_cursor=self.cursor)
    if callback:
      content = callback(content)
    self.respond_with_page(content, cursor, more) 

def make_query(self, ns):
    """Make a query of entities within this range.

    Query options are not supported. They should be specified when the query
    is run.

    Args:
      ns: namespace of this query.

    Returns:
      a db.Query or ndb.Query, depends on the model class's type.
    """
    if issubclass(self.model_class, db.Model):
      query = db.Query(self.model_class, namespace=ns)
      for f in self.filters:
        query.filter("%s %s" % (f[0], f[1]), f[2])
    else:
      query = self.model_class.query(namespace=ns)
      for f in self.filters:
        query = query.filter(ndb.FilterNode(*f))
    return query 

def PutResource(url, etag, content):
  """Persist a resource."""
  key = ndb.Key(Resource, url, namespace=settings.PLAYGROUND_NAMESPACE)
  keys = ndb.Query(ancestor=key).fetch(keys_only=True)
  ndb.delete_multi(keys)
  resource = Resource(id=url, etag=etag,
                      namespace=settings.PLAYGROUND_NAMESPACE)
  if len(content) <= _MAX_RAW_PROPERTY_BYTES:
    resource.content = content
    resource.put()
    return
  chunks = [content[i:i + _MAX_RAW_PROPERTY_BYTES]
            for i in range(0, len(content), _MAX_RAW_PROPERTY_BYTES)]
  entities = [ResourceChunk(id=i + 1, parent=resource.key, content=chunks[i])
              for i in range(0, len(chunks))]
  entities.append(resource)
  ndb.put_multi(entities) 

def make_ascending_datastore_query(self, kind, keys_only=False, filters=None):
    """Construct a query for this key range without setting the scan direction.

    Args:
      kind: A string.
      keys_only: bool, default False, use keys_only on Query?
      filters: optional list of filters to apply to the query. Each filter is
        a tuple: (<property_name_as_str>, <query_operation_as_str>, <value>).
        User filters are applied first.

    Returns:
      A datastore.Query instance.
    """
    query = datastore.Query(kind,
                            namespace=self.namespace,
                            _app=self._app,
                            keys_only=keys_only)
    query.Order(("__key__", datastore.Query.ASCENDING))

    query = self.filter_datastore_query(query, filters=filters)
    return query 

def make_ascending_ndb_query(self, kind_class, keys_only=False, filters=None):
    """Construct an NDB query for this key range, without the scan direction.

    Args:
      kind_class: An ndb.Model subclass.
      keys_only: bool, default False, query only for keys.

    Returns:
      An ndb.Query instance.
    """
    assert issubclass(kind_class, ndb.Model)
    if keys_only:
      default_options = ndb.QueryOptions(keys_only=True)
    else:
      default_options = None
    query = kind_class.query(app=self._app,
                             namespace=self.namespace,
                             default_options=default_options)
    query = self.filter_ndb_query(query, filters=filters)
    query = query.order(kind_class._key)
    return query 

def make_ascending_query(self, kind_class, keys_only=False, filters=None):
    """Construct a query for this key range without setting the scan direction.

    Args:
      kind_class: A kind implementation class (a subclass of either
        db.Model or ndb.Model).
      keys_only: bool, default False, query only for keys.
      filters: optional list of filters to apply to the query. Each filter is
        a tuple: (<property_name_as_str>, <query_operation_as_str>, <value>).
        User filters are applied first.

    Returns:
      A db.Query or ndb.Query instance (corresponding to kind_class).
    """
    if ndb is not None:
      if issubclass(kind_class, ndb.Model):
        return self.make_ascending_ndb_query(
            kind_class, keys_only=keys_only, filters=filters)
    assert self._app is None, '_app is not supported for db.Query'
    query = db.Query(kind_class, namespace=self.namespace, keys_only=keys_only)
    query.order("__key__")

    query = self.filter_query(query, filters=filters)
    return query 

def make_directed_datastore_query(self, kind, keys_only=False):
    """Construct a query for this key range, including the scan direction.

    Args:
      kind: A string.
      keys_only: bool, default False, use keys_only on Query?

    Returns:
      A datastore.Query instance.

    Raises:
      KeyRangeError: if self.direction is not in (KeyRange.ASC, KeyRange.DESC).
    """
    direction = self.__get_direction(datastore.Query.ASCENDING,
                                     datastore.Query.DESCENDING)
    query = datastore.Query(kind, _app=self._app, keys_only=keys_only)
    query.Order(("__key__", direction))

    query = self.filter_datastore_query(query)
    return query 

def make_query(self, ns):
    """Make a query of entities within this range.

    Query options are not supported. They should be specified when the query
    is run.

    Args:
      ns: namespace of this query.

    Returns:
      a db.Query or ndb.Query, depends on the model class's type.
    """
    if issubclass(self.model_class, db.Model):
      query = db.Query(self.model_class, namespace=ns)
      for f in self.filters:
        query.filter("%s %s" % (f[0], f[1]), f[2])
    else:
      query = self.model_class.query(namespace=ns)
      for f in self.filters:
        query = query.filter(ndb.FilterNode(*f))
    return query 

def make_change_log_query(target=None, auth_db_rev=None):
  """Returns ndb.Query over AuthDBChange entities."""
  if auth_db_rev:
    ancestor = change_log_revision_key(auth_db_rev)
  else:
    ancestor = change_log_root_key()
  q = AuthDBChange.query(ancestor=ancestor)
  if target:
    if not TARGET_RE.match(target):
      raise ValueError('Invalid target: %r' % target)
    q = q.filter(AuthDBChange.target == target)
  q = q.order(-AuthDBChange.key)
  return q


### Code to snapshot initial state of AuthDB into *History. 

def grab_all(self, ancestor):
    """Returns dicts with all entities under given ancestor."""
    entities = {}
    def cb(key):
      # Skip AuthDBLogRev itself, it's not interesting.
      if key == ancestor:
        return
      as_str = []
      k = key
      while k and k != ancestor:
        as_str.append('%s:%s' % (k.kind(), k.id()))
        k = k.parent()
      entities['/'.join(as_str)] = {
        prop: val for prop, val in key.get().to_dict().items() if val
      }
    ndb.Query(ancestor=ancestor).map(cb, keys_only=True)
    return entities 

def make_change_log_query(target=None, auth_db_rev=None):
  """Returns ndb.Query over AuthDBChange entities."""
  if auth_db_rev:
    ancestor = change_log_revision_key(auth_db_rev)
  else:
    ancestor = change_log_root_key()
  q = AuthDBChange.query(ancestor=ancestor)
  if target:
    if not TARGET_RE.match(target):
      raise ValueError('Invalid target: %r' % target)
    q = q.filter(AuthDBChange.target == target)
  q = q.order(-AuthDBChange.key)
  return q


### Code to snapshot initial state of AuthDB into *History. 

def grab_all(self, ancestor):
    """Returns dicts with all entities under given ancestor."""
    entities = {}
    def cb(key):
      # Skip AuthDBLogRev itself, it's not interesting.
      if key == ancestor:
        return
      as_str = []
      k = key
      while k and k != ancestor:
        as_str.append('%s:%s' % (k.kind(), k.id()))
        k = k.parent()
      entities['/'.join(as_str)] = {
        prop: val for prop, val in key.get().to_dict().items() if val
      }
    ndb.Query(ancestor=ancestor).map(cb, keys_only=True)
    return entities 

def make_change_log_query(target=None, auth_db_rev=None):
  """Returns ndb.Query over AuthDBChange entities."""
  if auth_db_rev:
    ancestor = change_log_revision_key(auth_db_rev)
  else:
    ancestor = change_log_root_key()
  q = AuthDBChange.query(ancestor=ancestor)
  if target:
    if not TARGET_RE.match(target):
      raise ValueError('Invalid target: %r' % target)
    q = q.filter(AuthDBChange.target == target)
  q = q.order(-AuthDBChange.key)
  return q


### Code to snapshot initial state of AuthDB into *History. 

def make_change_log_query(target=None, auth_db_rev=None):
  """Returns ndb.Query over AuthDBChange entities."""
  if auth_db_rev:
    ancestor = change_log_revision_key(auth_db_rev)
  else:
    ancestor = change_log_root_key()
  q = AuthDBChange.query(ancestor=ancestor)
  if target:
    if not TARGET_RE.match(target):
      raise ValueError('Invalid target: %r' % target)
    q = q.filter(AuthDBChange.target == target)
  q = q.order(-AuthDBChange.key)
  return q


### Code to snapshot initial state of AuthDB into *History. 

def make_query(self, ns):
    """Make a query of entities within this range.

    Query options are not supported. They should be specified when the query
    is run.

    Args:
      ns: namespace of this query.

    Returns:
      a db.Query or ndb.Query, depends on the model class's type.
    """
    if issubclass(self.model_class, db.Model):
      query = db.Query(self.model_class, namespace=ns)
      for f in self.filters:
        query.filter("%s %s" % (f[0], f[1]), f[2])
    else:
      query = self.model_class.query(namespace=ns)
      for f in self.filters:
        query = query.filter(ndb.FilterNode(*f))
    return query 

def make_change_log_query(target=None, auth_db_rev=None):
  """Returns ndb.Query over AuthDBChange entities."""
  if auth_db_rev:
    ancestor = change_log_revision_key(auth_db_rev)
  else:
    ancestor = change_log_root_key()
  q = AuthDBChange.query(ancestor=ancestor)
  if target:
    if not TARGET_RE.match(target):
      raise ValueError('Invalid target: %r' % target)
    q = q.filter(AuthDBChange.target == target)
  q = q.order(-AuthDBChange.key)
  return q


### Code to snapshot initial state of AuthDB into *History. 

def grab_all(self, ancestor):
    """Returns dicts with all entities under given ancestor."""
    entities = {}
    def cb(key):
      # Skip AuthDBLogRev itself, it's not interesting.
      if key == ancestor:
        return
      as_str = []
      k = key
      while k and k != ancestor:
        as_str.append('%s:%s' % (k.kind(), k.id()))
        k = k.parent()
      entities['/'.join(as_str)] = {
        prop: val for prop, val in key.get().to_dict().items() if val
      }
    ndb.Query(ancestor=ancestor).map(cb, keys_only=True)
    return entities 

def _yield_pages_async(q, size):
  """Given a ndb.Query, yields ndb.Future that returns pages of results
  asynchronously.
  """
  next_cursor = [None]
  should_continue = [True]
  def fire(page, result):
    results, cursor, more = page.get_result()
    result.set_result(results)
    next_cursor[0] = cursor
    should_continue[0] = more

  while should_continue[0]:
    page_future = q.fetch_page_async(size, start_cursor=next_cursor[0])
    result_future = ndb.Future()
    page_future.add_immediate_callback(fire, page_future, result_future)
    yield result_future
    result_future.get_result() 

def task_delete_tasks(task_ids):
  """Deletes the specified tasks, a list of string encoded task ids."""
  total = 0
  count = 0
  opt = ndb.QueryOptions(use_cache=False, use_memcache=False, keys_only=True)
  try:
    for task_id in task_ids:
      request_key = task_pack.unpack_request_key(task_id)
      # Delete the whole group. An ancestor query will retrieve the entity
      # itself too, so no need to explicitly delete it.
      keys = ndb.Query(default_options=opt, ancestor=request_key).fetch()
      if not keys:
        # Can happen if it is a retry.
        continue
      ndb.delete_multi(keys)
      total += len(keys)
      count += 1
    return count
  finally:
    logging.info(
        'Deleted %d TaskRequest groups; %d entities in total', count, total) 

def _IsNdbQuery(query):
  return ndb is not None and isinstance(query, ndb.Query) 

def _CoerceQueryParam(field, query_param):
  """Attempts to coerce `query_param` to match the ndb type of `field`.

  Args:
    field: The ndb field being queried.
    query_param: The query term to be coerced.

  Returns:
    The query param coerced if a coercion was possible.

  Raises:
    QueryTypeError: If there is an error with the type conversion.
  """
  if isinstance(field, ndb.IntegerProperty):
    try:
      return int(query_param)
    except ValueError:
      raise QueryTypeError(
          'Query param "%s" could not be converted to integer' % query_param)
  elif isinstance(field, ndb.BooleanProperty):
    if query_param.lower() == 'true':
      return True
    elif query_param.lower() == 'false':
      return False
    else:
      raise QueryTypeError(
          'Query param "%s" could not be converted to boolean' % query_param)
  elif isinstance(field, ndb.KeyProperty):
    key = datastore_utils.GetKeyFromUrlsafe(query_param)
    if not key:
      raise QueryTypeError(
          'Query param "%s" could not be converted to ndb.Key' % query_param)
    return key
  else:
    return query_param 

def _build_query_components(self):
    """Builds a dict containing a query and other useful objects.

    Returns:
      A dict containing the following keys:
        query: an NDB query with as many filters as can be made from the event's
            conditions. This will be limited by Datastore's limitation to having
            no more than one inequality filter.
        less_than_properties: a list of strings containing the names of
            properties in the event's conditions using the < or <= operators.
            This allows a cron job to filter entities that do not have these
            properties set, which unfortunately matches < for queries).
        extra_inequality_conditions: a list of inequality CustomEventConditions
            that could not be used because of the aforementioned Datastore
            limitation.
    """
    less_than_properties = []
    query_filters = []
    extra_inequality_conditions = []
    inequality_filter_seen = False

    for condition in self.conditions:
      if condition.opsymbol in ['<', '<=']:
        less_than_properties.append(condition.name)

      inequality_filter = condition.opsymbol in ['<', '<=', '!=', '>', '>=']

      if inequality_filter:
        if inequality_filter_seen:
          extra_inequality_conditions.append(condition)
          continue
        else:
          inequality_filter_seen = True

      query_filters.append(condition.get_filter())

    return {
        'query': ndb.Query(
            kind=self.model, filters=ndb.query.ConjunctionNode(*query_filters)),
        'less_than_properties': less_than_properties,
        'extra_inequality_conditions': extra_inequality_conditions} 

def Paginate(query, page_size=_SMALL_BATCH_SIZE, **query_options):
  """Performs the given query and breaks the results up into batches.

  Args:
    query: ndb.Query, the ndb query to paginate through and collect the results.
    page_size: int, The number of entities to request in each page.
    **query_options: dict, Any query option keyword args to pass to the query.

  Yields:
    Lists of results from the given query, at most page_size in length.
  """
  results = []
  cursor = None
  more = True

  while more:
    # Asynchronously fetch the next page.
    page_future = query.fetch_page_async(
        page_size, start_cursor=cursor, **query_options)

    # According to the docs, if more is True, there are *probably* more results.
    if results:
      yield results

    results, cursor, more = page_future.get_result()

  # Don't forget the last batch.
  if results:
    yield results 

def main(project_id):
    remote_api_stub.ConfigureRemoteApiForOAuth(
        '{}.appspot.com'.format(project_id),
        '/_ah/remote_api')

    # List the first 10 keys in the datastore.
    keys = ndb.Query().fetch(10, keys_only=True)

    for key in keys:
        print(key) 

def GetResource(url):
  """Retrieve a previously stored resource."""
  key = ndb.Key(Resource, url, namespace=settings.PLAYGROUND_NAMESPACE)
  query = ndb.Query(ancestor=key)
  results = query.fetch()
  if not results:
    return None, None
  resource = results[0]
  if resource.content is not None:
    return resource.etag, resource.content
  content = ''
  for resource_chunk in results[1:]:
    content += resource_chunk.content
  return resource.etag, content 

def fetch_page(query, batch_size, cursor_str, **kwargs):
  """Fetches a page from a query.

  Arguments:
    query: ndb.Query.
    batch_size: Maximum number of items to return.
    cursor_str: query-dependent string encoded cursor to continue a previous
        search.

  Returns:
  - items
  - str encoded cursor if relevant or None.
  """
  assert isinstance(query, ndb.Query), query
  if not 0 < batch_size <= 1000 or not isinstance(batch_size, int):
    raise ValueError(
        'batch_size must be between 1 and 1000, got %r', batch_size)
  if cursor_str:
    if not isinstance(cursor_str, basestring):
      raise ValueError(
          'cursor must be between valid string, got %r', cursor_str)
    cursor = datastore_query.Cursor(urlsafe=cursor_str)
  else:
    cursor = None
  items, cursor, more = query.fetch_page(
      batch_size, start_cursor=cursor, **kwargs)
  if not more:
    return items, None
  return items, cursor.urlsafe() 

def make_directed_ndb_query(self, kind_class, keys_only=False):
    """Construct an NDB query for this key range, including the scan direction.

    Args:
      kind_class: An ndb.Model subclass.
      keys_only: bool, default False, use keys_only on Query?

    Returns:
      An ndb.Query instance.

    Raises:
      KeyRangeError: if self.direction is not in (KeyRange.ASC, KeyRange.DESC).
    """
    assert issubclass(kind_class, ndb.Model)
    if keys_only:
      default_options = ndb.QueryOptions(keys_only=True)
    else:
      default_options = None
    query = kind_class.query(app=self._app,
                             namespace=self.namespace,
                             default_options=default_options)
    query = self.filter_ndb_query(query)
    if self.__get_direction(True, False):
      query = query.order(kind_class._key)
    else:
      query = query.order(-kind_class._key)
    return query 

def filter_datastore_query(self, query, filters=None):
    """Add query filter to restrict to this key range.

    Args:
      query: A datastore.Query instance.
      filters: optional list of filters to apply to the query. Each filter is
        a tuple: (<property_name_as_str>, <query_operation_as_str>, <value>).
        User filters are applied first.

    Returns:
      The input query restricted to this key range.
    """
    assert isinstance(query, datastore.Query)

    if filters:
      for f in filters:
        query.update({"%s %s" % (f[0], f[1]): f[2]})

    if self.include_start:
      start_comparator = ">="
    else:
      start_comparator = ">"
    if self.include_end:
      end_comparator = "<="
    else:
      end_comparator = "<"
    if self.key_start:
      query.update({"__key__ %s" % start_comparator: self.key_start})
    if self.key_end:
      query.update({"__key__ %s" % end_comparator: self.key_end})
    return query 

def filter_ndb_query(self, query, filters=None):
    """Add query filter to restrict to this key range.

    Args:
      query: An ndb.Query instance.
      filters: optional list of filters to apply to the query. Each filter is
        a tuple: (<property_name_as_str>, <query_operation_as_str>, <value>).
        User filters are applied first.

    Returns:
      The input query restricted to this key range.
    """
    assert _IsNdbQuery(query)


    if filters:
      for f in filters:
        query = query.filter(ndb.FilterNode(*f))

    if self.include_start:
      start_comparator = ">="
    else:
      start_comparator = ">"
    if self.include_end:
      end_comparator = "<="
    else:
      end_comparator = "<"
    if self.key_start:
      query = query.filter(ndb.FilterNode("__key__",
                                          start_comparator,
                                          self.key_start))
    if self.key_end:
      query = query.filter(ndb.FilterNode("__key__",
                                          end_comparator,
                                          self.key_end))
    return query