com.google.appengine.api.taskqueue.QueueFactory Java Examples

@Test
public void testRetryLimitIsHonored() throws Exception {
    long numTimesToFail = 10;
    int retryLimit = 2;
    String key = "testRetryLimitIsHonored-" + System.currentTimeMillis();

    Queue queue = QueueFactory.getDefaultQueue();
    queue.add(withUrl("/_ah/retryTest")
        .param("testdata-key", key)
        .param("times-to-fail", String.valueOf(numTimesToFail))
        .retryOptions(RetryOptions.Builder.withTaskRetryLimit(retryLimit)));

    long expectedAttempts = (long) (retryLimit + 1);
    String countKey = RetryTestServlet.getInvocationCountKey(key);
    Long actualAttempts = waitForTestData(countKey, expectedAttempts);

    // Ideally this would be the assert, but when a task fails with 500, the test framework
    // cannot capture it for the attempt count.
    // assertEquals(expectedAttempts, actualAttempts);

    // Allow room for one task to fail with 500.
    assertTrue("Task retries lower than specified via withTaskRetryLimit()",
        actualAttempts == expectedAttempts || actualAttempts == expectedAttempts - 1);
}
 

@Test
public void testDeleteTasks_usesMultipleBatches() {
  Queue defaultQ = QueueFactory.getQueue("default");
  TaskOptions taskOptA = withUrl("/a").taskName("a");
  TaskOptions taskOptB = withUrl("/b").taskName("b");
  TaskOptions taskOptC = withUrl("/c").taskName("c");
  taskQueueUtils.enqueue(defaultQ, ImmutableList.of(taskOptA, taskOptB, taskOptC));
  assertThat(getQueueInfo("default").getTaskInfo()).hasSize(3);

  taskQueueUtils.deleteTasks(
      defaultQ,
      ImmutableList.of(
          new TaskHandle(taskOptA, "default"),
          new TaskHandle(taskOptB, "default"),
          new TaskHandle(taskOptC, "default")));
  assertThat(getQueueInfo("default").getTaskInfo()).hasSize(0);
}
 

@Test
public void testSuccess_corruptTaskNoTld_discarded() {
  dnsQueue.addDomainRefreshTask("domain.com");
  dnsQueue.addDomainRefreshTask("domain.example");
  QueueFactory.getQueue(DNS_PULL_QUEUE_NAME)
      .add(
          TaskOptions.Builder.withDefaults()
              .method(Method.PULL)
              .param(DNS_TARGET_TYPE_PARAM, TargetType.DOMAIN.toString())
              .param(DNS_TARGET_NAME_PARAM, "domain.net"));

  run();

  // The corrupt task isn't in the pull queue, but also isn't in the push queue
  assertNoTasksEnqueued(DNS_PULL_QUEUE_NAME);
  assertTldsEnqueuedInPushQueue(
      ImmutableMultimap.of("com", "comWriter", "example", "exampleWriter"));
}
 

@Test
public void testSuccess_corruptTaskWrongType_discarded() {
  dnsQueue.addDomainRefreshTask("domain.com");
  dnsQueue.addDomainRefreshTask("domain.example");
  QueueFactory.getQueue(DNS_PULL_QUEUE_NAME)
      .add(
          TaskOptions.Builder.withDefaults()
              .method(Method.PULL)
              .param(DNS_TARGET_TYPE_PARAM, "Wrong type")
              .param(DNS_TARGET_NAME_PARAM, "domain.net")
              .param(PARAM_TLD, "net"));

  run();

  // The corrupt task isn't in the pull queue, but also isn't in the push queue
  assertNoTasksEnqueued(DNS_PULL_QUEUE_NAME);
  assertTldsEnqueuedInPushQueue(
      ImmutableMultimap.of("com", "comWriter", "example", "exampleWriter"));
}
 

@Test
public void testSuccess_corruptTaskNoName_discarded() {
  dnsQueue.addDomainRefreshTask("domain.com");
  dnsQueue.addDomainRefreshTask("domain.example");
  QueueFactory.getQueue(DNS_PULL_QUEUE_NAME)
      .add(
          TaskOptions.Builder.withDefaults()
              .method(Method.PULL)
              .param(DNS_TARGET_TYPE_PARAM, TargetType.DOMAIN.toString())
              .param(PARAM_TLD, "net"));

  run();

  // The corrupt task isn't in the pull queue, but also isn't in the push queue
  assertNoTasksEnqueued(DNS_PULL_QUEUE_NAME);
  assertTldsEnqueuedInPushQueue(
      ImmutableMultimap.of("com", "comWriter", "example", "exampleWriter"));
}
 

@Test
public void testRequestHeaders() throws Exception {
    String name = "testRequestHeaders-1-" + System.currentTimeMillis();
    Queue defaultQueue = QueueFactory.getDefaultQueue();
    defaultQueue.add(withTaskName(name));
    sync();

    RequestData request = DefaultQueueServlet.getLastRequest();
    assertEquals("default", request.getHeader(QUEUE_NAME));
    assertEquals(name, request.getHeader(TASK_NAME));
    assertNotNull(request.getHeader(TASK_RETRY_COUNT));
    assertNotNull(request.getHeader(TASK_EXECUTION_COUNT));
    assertNotNull(request.getHeader(TASK_ETA));

    String name2 = "testRequestHeaders-2-" + System.currentTimeMillis();
    Queue testQueue = QueueFactory.getQueue("test");
    testQueue.add(withTaskName(name2));
    sync();

    request = TestQueueServlet.getLastRequest();
    assertEquals("test", request.getHeader(QUEUE_NAME));
    assertEquals(name2, request.getHeader(TASK_NAME));
}
 

/**
 * Registers an update-replay-profile task.
 * @param sha1 SHA-1 of the file
 * @param incCommentsCount tells if comments count has to be incremented
 * @param incGgsCount      tells if GG count has to be incremented
 * @param incBgsCount      tells if BG count has to be incremented
 */
public static void register_updateRepProfileTask( final String sha1, final boolean incCommentsCount, final boolean incGgsCount, final boolean incBgsCount ) {
	int entitySelector;
	try {
		// SHA-1 algorithm is designed so every bit of SHA-1 depends on the input,
		// so the first byte depends on the input (and changes) just like all of it
		// It's enough to decide/assign the queue based on the first byte
		entitySelector = Integer.parseInt( sha1.substring( 0, 2 ), 16 );
	} catch ( final Exception e ) { // NumberFormatException is not enough, IndexOutOfBoundsException is raised if sha1 is shorter than 2 chars
		// This should never happen, but since sha1 is not checked syntactically,
		// it might happen someone sends an invalid sha1
		entitySelector = sha1.hashCode();
	}
	
	QueueFactory.getQueue( getEntityUpdateQueueName( entitySelector ) ).add(
		buildTask( OPERATION_UPDATE_REP_PROFILE )
			.param( PARAM_SHA1              , sha1 )
			.param( PARAM_INC_COMMENTS_COUNT, Boolean.toString( incCommentsCount ) )
			.param( PARAM_INC_GGS_COUNT     , Boolean.toString( incGgsCount      ) )
			.param( PARAM_INC_BGS_COUNT     , Boolean.toString( incBgsCount      ) )
	);
}
 

@Test
public void testMultiValueParams() throws Exception {
    class ParamHandler implements PrintServlet.RequestHandler {
        private String[] paramValues;

        public void handleRequest(ServletRequest req) {
            paramValues = req.getParameterValues("multi_value");
        }
    }

    ParamHandler handler = new ParamHandler();
    PrintServlet.setRequestHandler(handler);

    final Queue queue = QueueFactory.getQueue("tasks-queue");
    waitOnFuture(queue.addAsync(
        withUrl(URL)
            .param("multi_value", "param_value1")
            .param("multi_value", "param_value2")));
    sync();

    assertNotNull(handler.paramValues);
    assertEquals(
        new HashSet<String>(Arrays.asList("param_value1", "param_value2")),
        new HashSet<String>(Arrays.asList(handler.paramValues)));
}
 

@Test
public void testParams() throws Exception {
    class ParamHandler implements PrintServlet.RequestHandler {
        private String paramValue;

        public void handleRequest(ServletRequest req) {
            paramValue = req.getParameter("single_value");
        }
    }

    ParamHandler handler = new ParamHandler();
    PrintServlet.setRequestHandler(handler);

    final Queue queue = QueueFactory.getQueue("tasks-queue");
    queue.add(withUrl(URL).param("single_value", "param_value"));
    sync();

    assertEquals("param_value", handler.paramValue);
}
 

@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {

  // Increment the count for the current namespace asynchronously.
  QueueFactory.getDefaultQueue()
      .add(TaskOptions.Builder.withUrl("/_ah/update_count").param("countName", "SomeRequest"));
  // Increment the global count and set the
  // namespace locally.  The namespace is
  // transferred to the invoked request and
  // executed asynchronously.
  String namespace = NamespaceManager.get();
  try {
    NamespaceManager.set("-global-");
    QueueFactory.getDefaultQueue()
        .add(TaskOptions.Builder.withUrl("/_ah/update_count").param("countName", "SomeRequest"));
  } finally {
    NamespaceManager.set(namespace);
  }
  resp.setContentType("text/plain");
  resp.getWriter().println("Counts are being updated.");
}
 

@Test
public void testMultiValueParams() throws Exception {
    class ParamHandler implements PrintServlet.RequestHandler {
        private String[] paramValues;

        public void handleRequest(ServletRequest req) {
            paramValues = req.getParameterValues("multi_value");
        }
    }

    ParamHandler handler = new ParamHandler();
    PrintServlet.setRequestHandler(handler);

    final Queue queue = QueueFactory.getQueue("tasks-queue");
    queue.add(
        withUrl(URL)
            .param("multi_value", "param_value1")
            .param("multi_value", "param_value2"));
    sync();

    assertNotNull(handler.paramValues);
    assertEquals(
        new HashSet<>(Arrays.asList("param_value1", "param_value2")),
        new HashSet<>(Arrays.asList(handler.paramValues)));
}
 

@Test
public void testDeferredUserNS() {
    String testMethodTag = "testDeferredUserNS";
    String specifiedNameSpace = "the_testDeferredUserNS";
    Map<String, String> paramMap = dsUtil.createParamMap(testMethodTag);

    NamespaceManager.set(specifiedNameSpace);

    TaskOptions taskOptions = TaskOptions.Builder.withPayload(new ExecDeferred(dsUtil, paramMap));

    // no task name specified.
    QueueFactory.getQueue(E2E_TESTING_DEFERRED).add(taskOptions);
    Entity entity = dsUtil.waitForTaskThenFetchEntity(waitInterval, retryMax, testMethodTag);

    Map<String, String> expectedMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
    expectedMap.putAll(paramMap);
    expectedMap.put("X-AppEngine-Current-Namespace", specifiedNameSpace);

    dsUtil.assertTaskParamsMatchEntityProperties(expectedMap, entity);
}
 

@Test
public void testRetryLimitIsHonored() throws Exception {
    long numTimesToFail = 10;
    int retryLimit = 2;
    String key = "testRetryLimitIsHonored-" + System.currentTimeMillis();

    Queue queue = QueueFactory.getDefaultQueue();
    waitOnFuture(queue.addAsync(withUrl("/_ah/retryTest")
        .param("testdata-key", key)
        .param("times-to-fail", String.valueOf(numTimesToFail))
        .retryOptions(RetryOptions.Builder.withTaskRetryLimit(retryLimit))));

    Long expectedAttempts = (long) (retryLimit + 1);
    String countKey = RetryTestServlet.getInvocationCountKey(key);
    Long actualAttempts = waitForTestData(countKey, expectedAttempts);

    // Ideally this would be the assert, but when a task fails with 500, the test framework
    // cannot capture it for the attempt count.
    // assertEquals(expectedAttempts, actualAttempts);

    // Allow room for one task to fail with 500.
    assertTrue("Task retries lower than specified via withTaskRetryLimit()",
        actualAttempts == expectedAttempts || actualAttempts == expectedAttempts - 1);
}
 

@Test
public void testDeferredTaskNameSpecified() {
    String taskName = "This_is_my_deferred_task_name_" + testRunId;
    String testMethodTag = "testDeferredTaskNameSpecified";
    Map<String, String> paramMap = dsUtil.createParamMap(testMethodTag);

    TaskOptions taskOptions = TaskOptions.Builder.withTaskName(taskName).payload(new ExecDeferred(dsUtil, paramMap));

    QueueFactory.getQueue(E2E_TESTING_DEFERRED).add(taskOptions);
    Entity entity = dsUtil.waitForTaskThenFetchEntity(waitInterval, retryMax, testMethodTag);

    Map<String, String> expectedMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
    expectedMap.putAll(paramMap);
    expectedMap.put("X-AppEngine-TaskName", taskName);

    dsUtil.assertTaskParamsMatchEntityProperties(expectedMap, entity);
}
 

@Test
public void testUserNameSpace() {
    String testMethodTag = "testUserNameSpace";
    NamespaceManager.set("junittest");

    TaskOptions taskOptions = TaskOptions.Builder
        .withMethod(TaskOptions.Method.POST)
        .param(TEST_RUN_ID, testRunId)
        .param(TEST_METHOD_TAG, testMethodTag)
        .url("/queuetask/addentity");
    // task name explicitly not specified.

    QueueFactory.getQueue(E2E_TESTING).add(taskOptions);
    Entity entity = dsUtil.waitForTaskThenFetchEntity(waitInterval, retryMax, testMethodTag);
    Map<String, String> expectedParams = dsUtil.createParamMap(testMethodTag);
    dsUtil.assertTaskParamsMatchEntityProperties(expectedParams, entity);
}
 

@Test
public void testRetryOptionViaConfigFile() {
    String testMethodTag = "testRetryOptionViaConfigFile";

    TaskOptions taskOptions = TaskOptions.Builder
        .withMethod(TaskOptions.Method.POST)
        .param(TEST_RUN_ID, testRunId)
        .param(TEST_METHOD_TAG, testMethodTag)
        .url("/queuetask/addentity");

    // retry param. are defined in queue.xml
    QueueFactory.getQueue(E2E_TESTING_RETRY).add(taskOptions);
    Entity entity = dsUtil.waitForTaskThenFetchEntity(waitInterval, retryMax, testMethodTag);
    Map<String, String> expectedParams = dsUtil.createParamMap(testMethodTag);
    dsUtil.assertTaskParamsMatchEntityProperties(expectedParams, entity);
}
 

@Test
public void testRetryOption() {
    String testMethodTag = "testRetryOption";
    RetryOptions retryOptions = new RetryOptions(RetryOptions.Builder.withDefaults())
        .taskRetryLimit(5)
        .taskAgeLimitSeconds(10)
        .maxBackoffSeconds(10)
        .maxDoublings(10)
        .minBackoffSeconds(10);

    TaskOptions taskOptions = TaskOptions.Builder
        .withMethod(TaskOptions.Method.POST)
        .param(TEST_RUN_ID, testRunId)
        .param(TEST_METHOD_TAG, testMethodTag)
        .retryOptions(retryOptions)
        .url("/queuetask/addentity");

    QueueFactory.getQueue(E2E_TESTING).add(taskOptions);
    Entity entity = dsUtil.waitForTaskThenFetchEntity(waitInterval, retryMax, testMethodTag);
    Map<String, String> expectedParams = dsUtil.createParamMap(testMethodTag);
    dsUtil.assertTaskParamsMatchEntityProperties(expectedParams, entity);
}
 

@Test
public void testTaskNameSpecified() {
    // Add Task with specified name.
    String taskName = "This_is_my_task_name_" + testRunId;
    String testMethodTag = "testTaskNameSpecified";
    TaskOptions optionsHasName = TaskOptions.Builder
        .withMethod(TaskOptions.Method.POST)
        .param(TEST_RUN_ID, testRunId)
        .param(TEST_METHOD_TAG, testMethodTag)
        .taskName(taskName)
        .url("/queuetask/addentity")
        .etaMillis(0);

    QueueFactory.getQueue(E2E_TESTING).add(optionsHasName);
    Entity entity = dsUtil.waitForTaskThenFetchEntity(waitInterval, retryMax, testMethodTag);
    Map<String, String> expectedParams = dsUtil.createParamMap(testMethodTag);
    expectedParams.put("X-AppEngine-TaskName", taskName);
    dsUtil.assertTaskParamsMatchEntityProperties(expectedParams, entity);
}
 

private void doPolling() {
  Queue notificationQueue = QueueFactory.getQueue("notification-delivery");

  Worker worker = new Worker(notificationQueue);
  while (!LifecycleManager.getInstance().isShuttingDown()) {
    boolean tasksProcessed = worker.processBatchOfTasks();
    ApiProxy.flushLogs();

    if (!tasksProcessed) {
      // Wait before trying to lease tasks again.
      try {
        Thread.sleep(MILLISECONDS_TO_WAIT_WHEN_NO_TASKS_LEASED);
      } catch (InterruptedException e) {
        return;
      }
    }
  }

  log.info("Instance is shutting down");
}
 

private void doPolling() {
  Queue notificationQueue = QueueFactory.getQueue("notification-delivery");

  PushNotificationWorker worker = new PushNotificationWorker(notificationQueue);
  while (!LifecycleManager.getInstance().isShuttingDown()) {
    boolean tasksProcessed = worker.processBatchOfTasks();
    ApiProxy.flushLogs();

    if (!tasksProcessed) {
      // Wait before trying to lease tasks again.
      try {
        Thread.sleep(MILLISECONDS_TO_WAIT_WHEN_NO_TASKS_LEASED);
      } catch (InterruptedException e) {
        return;
      }
    }
  }

  log.info("Instance is shutting down");
}
 

@Test
public void testRequestHeaders() throws Exception {
    String name = "testRequestHeaders-1-" + System.currentTimeMillis();
    Queue defaultQueue = QueueFactory.getDefaultQueue();
    waitOnFuture(defaultQueue.addAsync(withTaskName(name)));
    sync();

    RequestData request = DefaultQueueServlet.getLastRequest();
    assertEquals("default", request.getHeader(QUEUE_NAME));
    assertEquals(name, request.getHeader(TASK_NAME));
    assertNotNull(request.getHeader(TASK_RETRY_COUNT));
    assertNotNull(request.getHeader(TASK_EXECUTION_COUNT));
    assertNotNull(request.getHeader(TASK_ETA));

    String name2 = "testRequestHeaders-2-" + System.currentTimeMillis();
    Queue testQueue = QueueFactory.getQueue("test");
    waitOnFuture(testQueue.addAsync(withTaskName(name2)));
    sync();

    request = TestQueueServlet.getLastRequest();
    assertEquals("test", request.getHeader(QUEUE_NAME));
    assertEquals(name2, request.getHeader(TASK_NAME));
}
 

@Test
public void testDeferredMarkForRetry() {
    String testMethodTag = "testDeferredWithRetry";
    Map<String, String> paramMap = dsUtil.createParamMap(testMethodTag);

    TaskOptions taskOptions = TaskOptions.Builder.withPayload(ExecDeferred.markForRetry(dsUtil, paramMap));

    QueueFactory.getQueue(E2E_TESTING_DEFERRED).add(taskOptions);
    Entity entity = dsUtil.waitForTaskThenFetchEntity(waitInterval, retryMax, testMethodTag);

    dsUtil.assertTaskParamsMatchEntityProperties(paramMap, entity);
}
 

@Test
public void testLeaseTasksOnlyReturnsSpecifiedNumberOfTasks() {
    Queue queue = QueueFactory.getQueue("pull-queue");
    TaskHandle th1 = queue.add(withMethod(PULL));
    TaskHandle th2 = queue.add(withMethod(PULL));
    sync(2000);
    try {
        int countLimit = 1;
        List<TaskHandle> handles = waitOnFuture(queue.leaseTasksAsync(10, TimeUnit.SECONDS, countLimit));
        assertEquals(countLimit, handles.size());
    } finally {
        queue.deleteTask(th1);
        queue.deleteTask(th2);
    }
}
 

@Test
public void testPayload() throws Exception {
    String sentPayload = "payload";

    Queue queue = QueueFactory.getDefaultQueue();
    queue.add(withPayload(sentPayload));
    sync();

    String receivedPayload = new String(DefaultQueueServlet.getLastRequest().getBody(), "UTF-8");
    assertEquals(sentPayload, receivedPayload);
}
 

/**
 * Asserts that the {@code numExpectedTasksInQueue} matches the actual number of tasks in the queue.
 */
private void assertNumTasksInQueue(int numExpectedTasksInQueue)
{
	LocalTaskQueue ltq = LocalTaskQueueTestConfig.getLocalTaskQueue();
	QueueStateInfo qsi = ltq.getQueueStateInfo()
		.get(QueueFactory.getQueue(DELETE_COUNTER_SHARD_QUEUE_NAME).getQueueName());
	assertEquals(numExpectedTasksInQueue, qsi.getTaskInfo().size());
}
 

@Test
public void testPullParams() throws Exception {
    final Queue queue = QueueFactory.getQueue("pull-queue");
    TaskHandle th = queue.add(withMethod(PULL).param("foo", "bar").etaMillis(15000));
    sync();
    try {
        List<TaskHandle> handles = queue.leaseTasks(30, TimeUnit.MINUTES, 100);
        assertFalse(handles.isEmpty());
        TaskHandle lh = handles.get(0);
        assertEquals(th.getName(), lh.getName());
    } finally {
        queue.deleteTask(th);
    }
}
 

@Test
public void testDeferredDefault() {
    String testMethodTag = "testDefault";
    Map<String, String> paramMap = dsUtil.createParamMap(testMethodTag);

    TaskOptions taskOptions = TaskOptions.Builder.withPayload(new ExecDeferred(dsUtil, paramMap));

    QueueFactory.getDefaultQueue().add(taskOptions);
    Entity entity = dsUtil.waitForTaskThenFetchEntity(waitInterval, retryMax, testMethodTag);

    dsUtil.assertTaskParamsMatchEntityProperties(paramMap, entity);
}
 

@Test
public void testHeaders() throws Exception {
    Queue queue = QueueFactory.getDefaultQueue();
    queue.add(withHeader("header_key", "header_value"));
    sync();

    RequestData lastRequest = DefaultQueueServlet.getLastRequest();
    assertEquals("header_value", lastRequest.getHeader("header_key"));
}
 

@Test
public void testExecQueue() {
    String testMethodTag = "testDefaultTag";  // Each test tagged for DS entity.
    TaskOptions taskoptions = TaskOptions.Builder
        .withMethod(TaskOptions.Method.POST)
        .param(TEST_RUN_ID, testRunId)  // testRunId used to track test in DS.
        .param(TEST_METHOD_TAG, testMethodTag)
        .etaMillis(0);

    QueueFactory.getQueue(E2E_TESTING_EXEC).add(taskoptions);
    Entity entity = dsUtil.waitForTaskThenFetchEntity(waitInterval, retryMax,
        testMethodTag);
    Map<String, String> expectedParams = dsUtil.createParamMap(testMethodTag);
    dsUtil.assertTaskParamsMatchEntityProperties(expectedParams, entity);
}
 

private void run() {
  ReadDnsQueueAction action = new ReadDnsQueueAction();
  action.tldUpdateBatchSize = TEST_TLD_UPDATE_BATCH_SIZE;
  action.requestedMaximumDuration = Duration.standardSeconds(10);
  action.clock = clock;
  action.dnsQueue = dnsQueue;
  action.dnsPublishPushQueue = QueueFactory.getQueue(DNS_PUBLISH_PUSH_QUEUE_NAME);
  action.hashFunction = Hashing.murmur3_32();
  action.taskQueueUtils = new TaskQueueUtils(new Retrier(null, 1));
  action.jitterSeconds = Optional.empty();
  // Advance the time a little, to ensure that leaseTasks() returns all tasks.
  clock.advanceBy(Duration.standardHours(1));

  action.run();
}