com.google.appengine.api.datastore.QueryResultIterator Java Examples

@Test
public void testIndexListFromIterator() throws Exception {
    Entity parent = createTestEntityWithUniqueMethodNameKey("Person", "testKeysOnly");
    Key key = parent.getKey();

    Entity tom = createEntity("Person", key)
        .withProperty("name", "Tom")
        .withProperty("surname", "Foe")
        .store();

    Query query = new Query("Person")
        .setAncestor(key)
        .setKeysOnly();

    PreparedQuery preparedQuery = service.prepare(query);
    QueryResultIterator<Entity> iter = preparedQuery.asQueryResultIterator();
    List<Index> indexes = iter.getIndexList();
    if (indexes != null) {
        // TODO -- something useful
        System.out.println("indexes = " + indexes);
    }
}
 

@Override
public Result<Book> listBooks(String startCursorString) {
  FetchOptions fetchOptions = FetchOptions.Builder.withLimit(10); // Only show 10 at a time
  if (startCursorString != null && !startCursorString.equals("")) {
    fetchOptions.startCursor(Cursor.fromWebSafeString(startCursorString)); // Where we left off
  }
  Query query = new Query(BOOK_KIND) // We only care about Books
      .addSort(Book.TITLE, SortDirection.ASCENDING); // Use default Index "title"
  PreparedQuery preparedQuery = datastore.prepare(query);
  QueryResultIterator<Entity> results = preparedQuery.asQueryResultIterator(fetchOptions);

  List<Book> resultBooks = entitiesToBooks(results);     // Retrieve and convert Entities
  Cursor cursor = results.getCursor();              // Where to start next time
  if (cursor != null && resultBooks.size() == 10) {         // Are we paging? Save Cursor
    String cursorString = cursor.toWebSafeString();               // Cursors are WebSafe
    return new Result<>(resultBooks, cursorString);
  } else {
    return new Result<>(resultBooks);
  }
}
 

@Override
public Result<Book> listBooksByUser(String userId, String startCursorString) {
  FetchOptions fetchOptions = FetchOptions.Builder.withLimit(10); // Only show 10 at a time
  if (startCursorString != null && !startCursorString.equals("")) {
    fetchOptions.startCursor(Cursor.fromWebSafeString(startCursorString)); // Where we left off
  }
  Query query = new Query(BOOK_KIND) // We only care about Books
      // Only for this user
      .setFilter(new Query.FilterPredicate(
          Book.CREATED_BY_ID, Query.FilterOperator.EQUAL, userId))
      // a custom datastore index is required since you are filtering by one property
      // but ordering by another
      .addSort(Book.TITLE, SortDirection.ASCENDING);
  PreparedQuery preparedQuery = datastore.prepare(query);
  QueryResultIterator<Entity> results = preparedQuery.asQueryResultIterator(fetchOptions);

  List<Book> resultBooks = entitiesToBooks(results);     // Retrieve and convert Entities
  Cursor cursor = results.getCursor();              // Where to start next time
  if (cursor != null && resultBooks.size() == 10) {         // Are we paging? Save Cursor
    String cursorString = cursor.toWebSafeString();               // Cursors are WebSafe
    return new Result<>(resultBooks, cursorString);
  } else {
    return new Result<>(resultBooks);
  }
}
 

@Override
public Result<Book> listBooks(String startCursorString) {
  FetchOptions fetchOptions = FetchOptions.Builder.withLimit(10); // Only show 10 at a time
  if (startCursorString != null && !startCursorString.equals("")) {
    fetchOptions.startCursor(Cursor.fromWebSafeString(startCursorString)); // Where we left off
  }
  Query query = new Query(BOOK_KIND) // We only care about Books
      .addSort(Book.TITLE, SortDirection.ASCENDING); // Use default Index "title"
  PreparedQuery preparedQuery = datastore.prepare(query);
  QueryResultIterator<Entity> results = preparedQuery.asQueryResultIterator(fetchOptions);

  List<Book> resultBooks = entitiesToBooks(results);     // Retrieve and convert Entities
  Cursor cursor = results.getCursor();              // Where to start next time
  if (cursor != null && resultBooks.size() == 10) {         // Are we paging? Save Cursor
    String cursorString = cursor.toWebSafeString();               // Cursors are WebSafe
    return new Result<>(resultBooks, cursorString);
  } else {
    return new Result<>(resultBooks);
  }
}
 

@Override
public Result<Book> listBooks(String startCursorString) {
  FetchOptions fetchOptions = FetchOptions.Builder.withLimit(10); // Only show 10 at a time
  if (startCursorString != null && !startCursorString.equals("")) {
    fetchOptions.startCursor(Cursor.fromWebSafeString(startCursorString)); // Where we left off
  }
  Query query = new Query(BOOK_KIND) // We only care about Books
      .addSort(Book.TITLE, SortDirection.ASCENDING); // Use default Index "title"
  PreparedQuery preparedQuery = datastore.prepare(query);
  QueryResultIterator<Entity> results = preparedQuery.asQueryResultIterator(fetchOptions);

  List<Book> resultBooks = entitiesToBooks(results);     // Retrieve and convert Entities
  Cursor cursor = results.getCursor();              // Where to start next time
  if (cursor != null && resultBooks.size() == 10) {         // Are we paging? Save Cursor
    String cursorString = cursor.toWebSafeString();               // Cursors are WebSafe
    return new Result<>(resultBooks, cursorString);
  } else {
    return new Result<>(resultBooks);
  }
}
 

@Override
public Result<Book> listBooks(String startCursorString) {
  FetchOptions fetchOptions = FetchOptions.Builder.withLimit(10); // Only show 10 at a time
  if (startCursorString != null && !startCursorString.equals("")) {
    fetchOptions.startCursor(Cursor.fromWebSafeString(startCursorString)); // Where we left off
  }
  Query query = new Query(BOOK_KIND) // We only care about Books
      .addSort(Book.TITLE, SortDirection.ASCENDING); // Use default Index "title"
  PreparedQuery preparedQuery = datastore.prepare(query);
  QueryResultIterator<Entity> results = preparedQuery.asQueryResultIterator(fetchOptions);

  List<Book> resultBooks = entitiesToBooks(results);     // Retrieve and convert Entities
  Cursor cursor = results.getCursor();              // Where to start next time
  if (cursor != null && resultBooks.size() == 10) {         // Are we paging? Save Cursor
    String cursorString = cursor.toWebSafeString();               // Cursors are WebSafe
    return new Result<>(resultBooks, cursorString);
  } else {
    return new Result<>(resultBooks);
  }
}
 

@Override
public Result<Book> listBooksByUser(String userId, String startCursorString) {
  FetchOptions fetchOptions = FetchOptions.Builder.withLimit(10); // Only show 10 at a time
  if (startCursorString != null && !startCursorString.equals("")) {
    fetchOptions.startCursor(Cursor.fromWebSafeString(startCursorString)); // Where we left off
  }
  Query query = new Query(BOOK_KIND) // We only care about Books
      // Only for this user
      .setFilter(new Query.FilterPredicate(
          Book.CREATED_BY_ID, Query.FilterOperator.EQUAL, userId))
      // a custom datastore index is required since you are filtering by one property
      // but ordering by another
      .addSort(Book.TITLE, SortDirection.ASCENDING);
  PreparedQuery preparedQuery = datastore.prepare(query);
  QueryResultIterator<Entity> results = preparedQuery.asQueryResultIterator(fetchOptions);

  List<Book> resultBooks = entitiesToBooks(results);     // Retrieve and convert Entities
  Cursor cursor = results.getCursor();              // Where to start next time
  if (cursor != null && resultBooks.size() == 10) {         // Are we paging? Save Cursor
    String cursorString = cursor.toWebSafeString();               // Cursors are WebSafe
    return new Result<>(resultBooks, cursorString);
  } else {
    return new Result<>(resultBooks);
  }
}
 

@Test
public void testCursor() throws Exception {
    Entity parent = createTestEntityWithUniqueMethodNameKey("Person", "testKeysOnly");
    Key key = parent.getKey();

    Entity john = createEntity("Person", key)
        .withProperty("name", "John")
        .withProperty("surname", "Doe")
        .store();

    Query query = new Query("Person")
        .setAncestor(key)
        .setKeysOnly();

    PreparedQuery preparedQuery = service.prepare(query);
    QueryResultIterator<Entity> iter = preparedQuery.asQueryResultIterator();
    Assert.assertNotNull(iter.next());
    Cursor cursor = iter.getCursor();

    iter = service.prepare(query).asQueryResultIterator(FetchOptions.Builder.withStartCursor(cursor));
    Assert.assertFalse(iter.hasNext());
}
 

/**
 * Fetches entities in batches and puts them into the buffer.
 */
private void batchFetching() {
    Query<T> newQuery = this.query.limit(BUFFER_SIZE);
    if (this.cursor != null) {
        newQuery = newQuery.startAt(this.cursor);
    }
    QueryResultIterator<T> iterator = newQuery.iterator();

    boolean shouldContinue = false;
    while (iterator.hasNext()) {
        shouldContinue = true;
        this.buffer.offer(iterator.next());
    }

    if (shouldContinue) {
        this.cursor = iterator.getCursor();
    }
}
 

@Test
public void testReverse() {
    Query query = new Query(kindName, rootKey);
    query.addSort(Entity.KEY_RESERVED_PROPERTY);
    QueryResultIterator<Entity> iter = service.prepare(query).asQueryResultIterator();
    Entity e1 = iter.next();
    Entity e2 = iter.next();
    Cursor cursor = iter.getCursor();
    //reverse
    query = query.reverse();
    cursor = cursor.reverse();
    iter = service.prepare(query).asQueryResultIterator(FetchOptions.Builder.withStartCursor(cursor));
    Assert.assertEquals(e2, iter.next());
    Assert.assertEquals(e1, iter.next());
}
 

@Test
public void testAsQueryResultIteratorWithOptions() throws Exception {
    QueryResultIterator<Entity> iterator = preparedQuery.asQueryResultIterator(withDefaults());
    assertNotNull(iterator);
    assertTrue(iterator.hasNext());
    assertEquals(john, iterator.next());
}
 

@Test
public void testAsQueryResultIterator() throws Exception {
    QueryResultIterator<Entity> iterator = preparedQuery.asQueryResultIterator();
    assertNotNull(iterator);
    assertTrue(iterator.hasNext());
    assertEquals(john, iterator.next());
}
 

@Override
public Pair<? extends Iterable<JobRecord>, String> queryRootPipelines(String classFilter,
    String cursor, final int limit) {
  Query query = new Query(JobRecord.DATA_STORE_KIND);
  Filter filter = classFilter == null || classFilter.isEmpty() ? new FilterPredicate(
      ROOT_JOB_DISPLAY_NAME, GREATER_THAN, null)
      : new FilterPredicate(ROOT_JOB_DISPLAY_NAME, EQUAL, classFilter);
  query.setFilter(filter);
  final PreparedQuery preparedQuery = dataStore.prepare(query);
  final FetchOptions fetchOptions = FetchOptions.Builder.withDefaults();
  if (limit > 0) {
    fetchOptions.limit(limit + 1);
  }
  if (cursor != null) {
    fetchOptions.startCursor(Cursor.fromWebSafeString(cursor));
  }
  return tryFiveTimes(
      new Operation<Pair<? extends Iterable<JobRecord>, String>>("queryRootPipelines") {
        @Override
        public Pair<? extends Iterable<JobRecord>, String> call() {
          QueryResultIterator<Entity> entities =
              preparedQuery.asQueryResultIterable(fetchOptions).iterator();
          Cursor dsCursor = null;
          List<JobRecord> roots = new LinkedList<>();
          while (entities.hasNext()) {
            if (limit > 0 && roots.size() >= limit) {
              dsCursor = entities.getCursor();
              break;
            }
            JobRecord jobRecord = new JobRecord(entities.next());
            roots.add(jobRecord);
          }
          return Pair.of(roots, dsCursor == null ? null : dsCursor.toWebSafeString());
        }
      });
}
 

@Override
@SuppressWarnings("unchecked")
protected void doOperation() {
    log("Running " + getClass().getSimpleName() + "...");
    log("Preview: " + isPreview());

    Cursor cursor = readPositionOfCursorFromFile().orElse(null);
    if (cursor == null) {
        log("Start from the beginning");
    } else {
        log("Start from cursor position: " + cursor.toWebSafeString());
    }

    boolean shouldContinue = true;
    while (shouldContinue) {
        shouldContinue = false;
        Query<T> filterQueryKeys = getFilterQuery().limit(BATCH_SIZE);
        if (cursor != null) {
            filterQueryKeys = filterQueryKeys.startAt(cursor);
        }
        QueryResultIterator<?> iterator;
        if (shouldUseTransaction()) {
            iterator = filterQueryKeys.keys().iterator();
        } else {
            iterator = filterQueryKeys.iterator();
        }

        while (iterator.hasNext()) {
            shouldContinue = true;

            // migrate
            if (shouldUseTransaction()) {
                migrateWithTrx((Key<T>) iterator.next());
            } else {
                migrateWithoutTrx((T) iterator.next());
            }

            numberOfScannedKey.incrementAndGet();
        }

        if (shouldContinue) {
            cursor = iterator.getCursor();
            flushEntitiesSavingBuffer();
            savePositionOfCursorToFile(cursor);
            log(String.format("Cursor Position: %s", cursor.toWebSafeString()));
            log(String.format("Number Of Entity Key Scanned: %d", numberOfScannedKey.get()));
            log(String.format("Number Of Entity affected: %d", numberOfAffectedEntities.get()));
            log(String.format("Number Of Entity updated: %d", numberOfUpdatedEntities.get()));
        }
    }

    deleteCursorPositionFile();
    log(isPreview() ? "Preview Completed!" : "Migration Completed!");
    log("Total number of entities: " + numberOfScannedKey.get());
    log("Number of affected entities: " + numberOfAffectedEntities.get());
    log("Number of updated entities: " + numberOfUpdatedEntities.get());
}
 

@Override
public ListItemBatch list(String bucket, String prefix, String delimiter,
    String marker, int maxResults, long timeoutMillis) throws IOException {
  ensureInitialized();
  Query query = makeQuery(bucket);
  int prefixLength;
  if (!Strings.isNullOrEmpty(prefix)) {
    Key keyPrefix = makeKey(bucket, prefix);
    query.setFilter(new FilterPredicate(KEY_RESERVED_PROPERTY, GREATER_THAN_OR_EQUAL, keyPrefix));
    prefixLength = prefix.length();
  } else {
    prefixLength = 0;
  }
  FetchOptions fetchOptions = FetchOptions.Builder.withDefaults();
  if (marker != null) {
    fetchOptions.startCursor(Cursor.fromWebSafeString(marker));
  }
  List<ListItem> items = new ArrayList<>(maxResults);
  Set<String> prefixes = new HashSet<>();
  QueryResultIterator<Entity> dsResults =
      datastore.prepare(query).asQueryResultIterator(fetchOptions);
  while (items.size() < maxResults && dsResults.hasNext()) {
    Entity entity = dsResults.next();
    String name = entity.getKey().getName();
    if (prefixLength > 0 && !name.startsWith(prefix)) {
      break;
    }
    if (!Strings.isNullOrEmpty(delimiter)) {
      int delimiterIdx = name.indexOf(delimiter, prefixLength);
      if (delimiterIdx > 0) {
        name = name.substring(0, delimiterIdx + 1);
        if (prefixes.add(name)) {
          items.add(new ListItem.Builder().setName(name).setDirectory(true).build());
        }
        continue;
      }
    }
    GcsFilename filename = new GcsFilename(bucket, name);
    GcsFileMetadata metadata = createGcsFileMetadata(entity, filename);
    ListItem listItem = new ListItem.Builder()
        .setName(name)
        .setLength(metadata.getLength())
        .setLastModified(metadata.getLastModified())
        .build();
    items.add(listItem);
  }
  Cursor cursor = dsResults.getCursor();
  String nextMarker = null;
  if (items.size() == maxResults && cursor != null) {
    nextMarker = cursor.toWebSafeString();
  }
  return new ListItemBatch(items, nextMarker);
}
 

@Override
public QueryResultIterator<EppResourceIndex> getQueryIterator(@Nullable Cursor cursor) {
  return startQueryAt(query(), cursor).iterator();
}
 

/** Query for children of the current resource and of the current child class. */
@Override
public QueryResultIterator<I> getQueryIterator(Cursor cursor) {
  return startQueryAt(ofy().load().type(type).ancestor(ancestor), cursor).iterator();
}
 

@Override
public QueryResultIterator<Key<CommitLogManifest>> getQueryIterator(@Nullable Cursor cursor) {
  return startQueryAt(query(), cursor).keys().iterator();
}
 

/**
 * Return the iterator over Query results, starting at the cursor location.
 *
 * <p>Must always return an iterator over the same query.
 *
 * <p>The underlying {@link Query} must have an ancestor filter, so that it is strongly
 * consistent. According to the documentation at
 * https://cloud.google.com/appengine/docs/java/datastore/queries#Java_Data_consistency
 *
 * <p>"strongly consistent queries are always transactionally consistent". However, each time we
 * restart the query at a cursor we have a new effective query, and "if the results for a query
 * change between uses of a cursor, the query notices only changes that occur in results after the
 * cursor. If a new result appears before the cursor's position for the query, it will not be
 * returned when the results after the cursor are fetched."
 *
 * <p>What this means in practice is that entities that are created after the initial query begins
 * may or may not be seen by this reader, depending on whether the query was paused and restarted
 * with a cursor before it would have reached the new entity.
 *
 * @param cursor the initial location for the iterator to start from. If null - start from
 *     beginning.
 */
public abstract QueryResultIterator<I> getQueryIterator(@Nullable Cursor cursor);