build.bazel.remote.execution.v2.ExecuteRequest Java Examples

@Test
public void testServerLogsNotSavedForSuccessfulAction() throws Exception {
  RemoteSpawnRunner runner = newSpawnRunner();

  Digest logDigest = digestUtil.computeAsUtf8("bla");
  ActionResult result = ActionResult.newBuilder().setExitCode(0).build();
  when(executor.executeRemotely(any(ExecuteRequest.class)))
      .thenReturn(
          ExecuteResponse.newBuilder()
              .putServerLogs(
                  "logname",
                  LogFile.newBuilder().setHumanReadable(true).setDigest(logDigest).build())
              .setResult(result)
              .build());

  Spawn spawn = newSimpleSpawn();
  FakeSpawnExecutionContext policy = getSpawnContext(spawn);

  SpawnResult res = runner.exec(spawn, policy);
  assertThat(res.status()).isEqualTo(Status.SUCCESS);

  verify(executor).executeRemotely(any(ExecuteRequest.class));
  verify(cache).download(eq(result), eq(execRoot), any(FileOutErr.class), any());
  verify(cache, never()).downloadFile(any(Path.class), any(Digest.class));
}
 

@Test
public void testNoRetryOnOk() throws IOException {
  String uniqueName = InProcessServerBuilder.generateName();
  ExecutionImpl service = new ExecutionImpl(Status.OK, 0);
  InProcessServerBuilder.forName(uniqueName).addService(service).build().start();
  CallCounter beforeRetry = new CallCounter();
  ManagedChannel channel =
      InProcessChannelBuilder.forName(uniqueName)
          .intercept(
              new RetryClientInterceptor(
                  RetryPolicy.builder().setMaxRetries(2).setBeforeRetry(beforeRetry).build()))
          .build();
  ExecutionBlockingStub stub = ExecutionGrpc.newBlockingStub(channel);
  stub.execute(ExecuteRequest.newBuilder().build()).forEachRemaining(resp -> {});

  Assert.assertEquals(1, service.calls);
  Assert.assertEquals(0, beforeRetry.calls);
}
 

@Test
public void testRetryOnUnavailable() throws IOException {
  String uniqueName = InProcessServerBuilder.generateName();
  ExecutionImpl service = new ExecutionImpl(Status.UNAVAILABLE, 0);
  InProcessServerBuilder.forName(uniqueName).addService(service).build().start();
  CallCounter beforeRetry = new CallCounter();
  ManagedChannel channel =
      InProcessChannelBuilder.forName(uniqueName)
          .intercept(
              new RetryClientInterceptor(
                  RetryPolicy.builder().setMaxRetries(2).setBeforeRetry(beforeRetry).build()))
          .build();
  ExecutionBlockingStub stub = ExecutionGrpc.newBlockingStub(channel);
  try {
    stub.execute(ExecuteRequest.newBuilder().build()).forEachRemaining(resp -> {});
    Assert.fail("Final retry should cause an exception");
  } catch (StatusRuntimeException ex) {
    Assert.assertEquals(Status.Code.UNAVAILABLE, ex.getStatus().getCode());
  }

  Assert.assertEquals(3, service.calls);
  Assert.assertEquals(2, beforeRetry.calls);
}
 

@Test
public void testNonHumanReadableServerLogsNotSaved() throws Exception {
  RemoteSpawnRunner runner = newSpawnRunner();

  Digest logDigest = digestUtil.computeAsUtf8("bla");
  ActionResult result = ActionResult.newBuilder().setExitCode(31).build();
  when(executor.executeRemotely(any(ExecuteRequest.class)))
      .thenReturn(
          ExecuteResponse.newBuilder()
              .putServerLogs("logname", LogFile.newBuilder().setDigest(logDigest).build())
              .setResult(result)
              .build());

  Spawn spawn = newSimpleSpawn();
  FakeSpawnExecutionContext policy = getSpawnContext(spawn);
  SpawnResult res = runner.exec(spawn, policy);
  assertThat(res.status()).isEqualTo(Status.NON_ZERO_EXIT);

  verify(executor).executeRemotely(any(ExecuteRequest.class));
  verify(cache).download(eq(result), eq(execRoot), any(FileOutErr.class), any());
  verify(cache, never()).downloadFile(any(Path.class), any(Digest.class));
}
 

@Test
public void nonCachableSpawnsShouldNotBeCached_localFallback() throws Exception {
  // Test that if a non-cachable spawn is executed locally due to the local fallback,
  // that its result is not uploaded to the remote cache.

  remoteOptions.remoteAcceptCached = true;
  remoteOptions.remoteLocalFallback = true;
  remoteOptions.remoteUploadLocalResults = true;

  RemoteSpawnRunner runner = newSpawnRunner();

  // Throw an IOException to trigger the local fallback.
  when(executor.executeRemotely(any(ExecuteRequest.class))).thenThrow(IOException.class);

  Spawn spawn = simpleSpawnWithExecutionInfo(NO_CACHE);
  SpawnExecutionContext policy = getSpawnContext(spawn);

  runner.exec(spawn, policy);

  verify(localRunner).exec(spawn, policy);
  verify(cache).ensureInputsPresent(any(), any());
  verifyNoMoreInteractions(cache);
}
 

@Test
public void testDownloadMinimalOnCacheMiss() throws Exception {
  // arrange
  remoteOptions.remoteOutputsMode = RemoteOutputsMode.MINIMAL;

  ActionResult succeededAction = ActionResult.newBuilder().setExitCode(0).build();
  ExecuteResponse succeeded = ExecuteResponse.newBuilder().setResult(succeededAction).build();
  when(executor.executeRemotely(any(ExecuteRequest.class))).thenReturn(succeeded);

  RemoteSpawnRunner runner = newSpawnRunner();

  Spawn spawn = newSimpleSpawn();
  FakeSpawnExecutionContext policy = getSpawnContext(spawn);

  // act
  SpawnResult result = runner.exec(spawn, policy);
  assertThat(result.exitCode()).isEqualTo(0);
  assertThat(result.status()).isEqualTo(Status.SUCCESS);

  // assert
  verify(executor).executeRemotely(any());
  verify(cache)
      .downloadMinimal(
          any(), eq(succeededAction), anyCollection(), any(), any(), any(), any(), any());
  verify(cache, never()).download(any(ActionResult.class), any(Path.class), eq(outErr), any());
}
 

@Test
public void testExitCode_executorfailure() throws Exception {
  // If we get a failure due to the remote cache not working, the exit code should be
  // ExitCode.REMOTE_ERROR.

  remoteOptions.remoteLocalFallback = false;

  RemoteSpawnRunner runner = newSpawnRunner();

  when(cache.downloadActionResult(any(ActionKey.class), /* inlineOutErr= */ eq(false)))
      .thenReturn(null);
  when(executor.executeRemotely(any(ExecuteRequest.class))).thenThrow(new IOException("reasons"));

  Spawn spawn = newSimpleSpawn();
  SpawnExecutionContext policy = getSpawnContext(spawn);

  SpawnResult result = runner.exec(spawn, policy);
  assertThat(result.exitCode()).isEqualTo(ExitCode.REMOTE_ERROR.getNumericExitCode());
  assertThat(result.getDetailMessage("", "", false, false, false)).contains("reasons");
}
 

@Test
public void treatFailedCachedActionAsCacheMiss_remote() throws Exception {
  // Test that bazel treats failed cache action as a cache miss and attempts to execute action
  // remotely

  ActionResult failedAction = ActionResult.newBuilder().setExitCode(1).build();
  when(cache.downloadActionResult(any(ActionKey.class), /* inlineOutErr= */ eq(false)))
      .thenReturn(failedAction);

  RemoteSpawnRunner runner = newSpawnRunner();

  ExecuteResponse succeeded =
      ExecuteResponse.newBuilder()
          .setResult(ActionResult.newBuilder().setExitCode(0).build())
          .build();
  when(executor.executeRemotely(any(ExecuteRequest.class))).thenReturn(succeeded);
  Spawn spawn = newSimpleSpawn();
  SpawnExecutionContext policy = getSpawnContext(spawn);

  runner.exec(spawn, policy);

  ArgumentCaptor<ExecuteRequest> requestCaptor = ArgumentCaptor.forClass(ExecuteRequest.class);
  verify(executor).executeRemotely(requestCaptor.capture());
  assertThat(requestCaptor.getValue().getSkipCacheLookup()).isTrue();
}
 

@Test
public void testHumanReadableServerLogsSavedForFailingAction() throws Exception {
  RemoteSpawnRunner runner = newSpawnRunner();
  Digest logDigest = digestUtil.computeAsUtf8("bla");
  Path logPath = logDir.getRelative(simpleActionId).getRelative("logname");
  when(executor.executeRemotely(any(ExecuteRequest.class)))
      .thenReturn(
          ExecuteResponse.newBuilder()
              .putServerLogs(
                  "logname",
                  LogFile.newBuilder().setHumanReadable(true).setDigest(logDigest).build())
              .setResult(ActionResult.newBuilder().setExitCode(31).build())
              .build());
  SettableFuture<Void> completed = SettableFuture.create();
  completed.set(null);
  when(cache.downloadFile(eq(logPath), eq(logDigest))).thenReturn(completed);

  Spawn spawn = newSimpleSpawn();
  SpawnExecutionContext policy = getSpawnContext(spawn);

  SpawnResult res = runner.exec(spawn, policy);
  assertThat(res.status()).isEqualTo(Status.NON_ZERO_EXIT);

  verify(executor).executeRemotely(any(ExecuteRequest.class));
  verify(cache).downloadFile(eq(logPath), eq(logDigest));
}
 

@Test
public void noRemoteExecutorFallbackFails() throws Exception {
  // Errors from the fallback runner should be propagated out of the remote runner.

  remoteOptions.remoteUploadLocalResults = true;
  remoteOptions.remoteLocalFallback = true;

  RemoteSpawnRunner runner = newSpawnRunner();
  // Trigger local fallback
  when(executor.executeRemotely(any(ExecuteRequest.class))).thenThrow(new IOException());

  Spawn spawn = newSimpleSpawn();
  SpawnExecutionContext policy = getSpawnContext(spawn);

  when(cache.downloadActionResult(any(ActionKey.class), /* inlineOutErr= */ eq(false)))
      .thenReturn(null);

  IOException err = new IOException("local execution error");
  when(localRunner.exec(eq(spawn), eq(policy))).thenThrow(err);

  IOException e = assertThrows(IOException.class, () -> runner.exec(spawn, policy));
  assertThat(e).isSameInstanceAs(err);

  verify(localRunner).exec(eq(spawn), eq(policy));
}
 

@Test
public void remoteCacheErrorFallbackFails() throws Exception {
  // Errors from the fallback runner should be propagated out of the remote runner.

  remoteOptions.remoteUploadLocalResults = true;
  remoteOptions.remoteLocalFallback = true;

  RemoteSpawnRunner runner = newSpawnRunner();
  // Trigger local fallback
  when(executor.executeRemotely(any(ExecuteRequest.class))).thenThrow(new IOException());

  Spawn spawn = newSimpleSpawn();
  SpawnExecutionContext policy = getSpawnContext(spawn);

  when(cache.downloadActionResult(any(ActionKey.class), /* inlineOutErr= */ eq(false)))
      .thenThrow(new IOException());

  IOException err = new IOException("local execution error");
  when(localRunner.exec(eq(spawn), eq(policy))).thenThrow(err);

  IOException e = assertThrows(IOException.class, () -> runner.exec(spawn, policy));
  assertThat(e).isSameInstanceAs(err);

  verify(localRunner).exec(eq(spawn), eq(policy));
}
 

@Test
public void testLocalFallbackFailureRemoteExecutorFailure() throws Exception {
  remoteOptions.remoteLocalFallback = true;

  RemoteSpawnRunner runner = newSpawnRunner();

  when(cache.downloadActionResult(any(ActionKey.class), /* inlineOutErr= */ eq(false)))
      .thenReturn(null);
  when(executor.executeRemotely(any(ExecuteRequest.class))).thenThrow(new IOException());

  Spawn spawn = newSimpleSpawn();
  SpawnExecutionContext policy = getSpawnContext(spawn);

  IOException err = new IOException("local execution error");
  when(localRunner.exec(eq(spawn), eq(policy))).thenThrow(err);

  IOException e = assertThrows(IOException.class, () -> runner.exec(spawn, policy));
  assertThat(e).isSameInstanceAs(err);

  verify(localRunner).exec(eq(spawn), eq(policy));
}
 

@Test
public void resultsDownloadFailureTriggersRemoteExecutionWithSkipCacheLookup() throws Exception {
  // If downloading an action result fails, remote execution should be retried
  // with skip cache lookup enabled

  RemoteSpawnRunner runner = newSpawnRunner();

  when(cache.downloadActionResult(any(ActionKey.class), /* inlineOutErr= */ eq(false)))
      .thenReturn(null);
  ActionResult cachedResult = ActionResult.newBuilder().setExitCode(0).build();
  ActionResult execResult = ActionResult.newBuilder().setExitCode(31).build();
  ExecuteResponse cachedResponse =
      ExecuteResponse.newBuilder().setResult(cachedResult).setCachedResult(true).build();
  ExecuteResponse executedResponse = ExecuteResponse.newBuilder().setResult(execResult).build();
  when(executor.executeRemotely(any(ExecuteRequest.class)))
      .thenReturn(cachedResponse)
      .thenReturn(executedResponse);
  Exception downloadFailure =
      new BulkTransferException(new CacheNotFoundException(Digest.getDefaultInstance()));
  doThrow(downloadFailure)
      .when(cache)
      .download(eq(cachedResult), any(Path.class), any(FileOutErr.class), any());
  doNothing().when(cache).download(eq(execResult), any(Path.class), any(FileOutErr.class), any());

  Spawn spawn = newSimpleSpawn();

  SpawnExecutionContext policy = getSpawnContext(spawn);

  SpawnResult res = runner.exec(spawn, policy);
  assertThat(res.status()).isEqualTo(Status.NON_ZERO_EXIT);
  assertThat(res.exitCode()).isEqualTo(31);

  ArgumentCaptor<ExecuteRequest> requestCaptor = ArgumentCaptor.forClass(ExecuteRequest.class);
  verify(executor, times(2)).executeRemotely(requestCaptor.capture());
  List<ExecuteRequest> requests = requestCaptor.getAllValues();
  // first request should have been executed without skip cache lookup
  assertThat(requests.get(0).getSkipCacheLookup()).isFalse();
  // second should have been executed with skip cache lookup
  assertThat(requests.get(1).getSkipCacheLookup()).isTrue();
}
 

@Test
public void testExecuteCancel() throws Exception {
  AtomicReference<StreamObserver<Operation>> responseObserverCapture = new AtomicReference<>();

  services.add(
      new ExecutionImplBase() {
        @Override
        public void execute(ExecuteRequest request, StreamObserver<Operation> responseObserver) {
          responseObserverCapture.set(responseObserver);
        }
      });

  setupServer();

  ExecutionHandle executionResult =
      clients
          .getRemoteExecutionService()
          .execute(
              clients.getProtocol().computeDigest("".getBytes(Charsets.UTF_8)),
              "",
              MetadataProviderFactory.emptyMetadataProvider());

  responseObserverCapture.get().onNext(Operation.newBuilder().setDone(false).build());

  executionResult.cancel();

  // Check that the server is notified of cancellation.
  expectedException.expect(StatusRuntimeException.class);
  responseObserverCapture.get().onNext(Operation.newBuilder().setDone(false).build());
}
 

@Test
public void testRemoteExecutionTimeout() throws Exception {
  // If remote execution times out the SpawnResult status should be TIMEOUT.

  remoteOptions.remoteLocalFallback = false;

  RemoteSpawnRunner runner = newSpawnRunner();

  ActionResult cachedResult = ActionResult.newBuilder().setExitCode(0).build();
  when(cache.downloadActionResult(any(ActionKey.class), /* inlineOutErr= */ eq(false)))
      .thenReturn(null);
  ExecuteResponse resp =
      ExecuteResponse.newBuilder()
          .setResult(cachedResult)
          .setStatus(
              com.google.rpc.Status.newBuilder()
                  .setCode(Code.DEADLINE_EXCEEDED.getNumber())
                  .build())
          .build();
  when(executor.executeRemotely(any(ExecuteRequest.class)))
      .thenThrow(new IOException(new ExecutionStatusException(resp.getStatus(), resp)));

  Spawn spawn = newSimpleSpawn();

  SpawnExecutionContext policy = getSpawnContext(spawn);

  SpawnResult res = runner.exec(spawn, policy);
  assertThat(res.status()).isEqualTo(Status.TIMEOUT);

  verify(executor).executeRemotely(any(ExecuteRequest.class));
  verify(cache).download(eq(cachedResult), eq(execRoot), any(FileOutErr.class), any());
}
 

@Test
public void testRemoteExecutionTimeoutDoesNotTriggerFallback() throws Exception {
  // If remote execution times out the SpawnResult status should be TIMEOUT, regardess of local
  // fallback option.

  remoteOptions.remoteLocalFallback = true;

  RemoteSpawnRunner runner = newSpawnRunner();

  ActionResult cachedResult = ActionResult.newBuilder().setExitCode(0).build();
  when(cache.downloadActionResult(any(ActionKey.class), /* inlineOutErr= */ eq(false)))
      .thenReturn(null);
  ExecuteResponse resp =
      ExecuteResponse.newBuilder()
          .setResult(cachedResult)
          .setStatus(
              com.google.rpc.Status.newBuilder()
                  .setCode(Code.DEADLINE_EXCEEDED.getNumber())
                  .build())
          .build();
  when(executor.executeRemotely(any(ExecuteRequest.class)))
      .thenThrow(new IOException(new ExecutionStatusException(resp.getStatus(), resp)));

  Spawn spawn = newSimpleSpawn();

  SpawnExecutionContext policy = getSpawnContext(spawn);

  SpawnResult res = runner.exec(spawn, policy);
  assertThat(res.status()).isEqualTo(Status.TIMEOUT);

  verify(executor).executeRemotely(any(ExecuteRequest.class));
  verify(cache).download(eq(cachedResult), eq(execRoot), any(FileOutErr.class), any());
  verify(localRunner, never()).exec(eq(spawn), eq(policy));
}
 

@Test
public void testRemoteExecutionCommandFailureDoesNotTriggerFallback() throws Exception {
  remoteOptions.remoteLocalFallback = true;

  RemoteSpawnRunner runner = newSpawnRunner();

  ActionResult cachedResult = ActionResult.newBuilder().setExitCode(0).build();
  when(cache.downloadActionResult(any(ActionKey.class), /* inlineOutErr= */ eq(false)))
      .thenReturn(null);
  ExecuteResponse failed =
      ExecuteResponse.newBuilder()
          .setResult(ActionResult.newBuilder().setExitCode(33).build())
          .build();
  when(executor.executeRemotely(any(ExecuteRequest.class))).thenReturn(failed);

  Spawn spawn = newSimpleSpawn();

  SpawnExecutionContext policy = getSpawnContext(spawn);

  SpawnResult res = runner.exec(spawn, policy);
  assertThat(res.status()).isEqualTo(Status.NON_ZERO_EXIT);
  assertThat(res.exitCode()).isEqualTo(33);

  verify(executor).executeRemotely(any(ExecuteRequest.class));
  verify(cache, never()).download(eq(cachedResult), eq(execRoot), any(FileOutErr.class), any());
  verify(localRunner, never()).exec(eq(spawn), eq(policy));
}
 

@Override
public void execute(ExecuteRequest request, StreamObserver<Operation> responseObserver) {
  calls++;
  for (int i = 0; i < partialResponses; i++) {
    responseObserver.onNext(Operation.newBuilder().build());
  }
  responseObserver.onError(status.asRuntimeException());
}
 

@Test
public void testRetryOnPartialResponse() throws IOException {
  String uniqueName = InProcessServerBuilder.generateName();
  ExecutionImpl service = new ExecutionImpl(Status.UNAVAILABLE, 1);
  InProcessServerBuilder.forName(uniqueName).addService(service).build().start();
  CallCounter beforeRetry = new CallCounter();
  ManagedChannel channel =
      InProcessChannelBuilder.forName(uniqueName)
          .intercept(
              new RetryClientInterceptor(
                  RetryPolicy.builder()
                      .setMaxRetries(2)
                      .setBeforeRetry(beforeRetry)
                      .setRestartAllStreamingCalls(true)
                      .build()))
          .build();
  ExecutionBlockingStub stub = ExecutionGrpc.newBlockingStub(channel);
  try {
    stub.execute(ExecuteRequest.newBuilder().build()).forEachRemaining(resp -> {});
    Assert.fail("Final retry should cause an exception");
  } catch (StatusRuntimeException ex) {
    Assert.assertEquals(Status.Code.UNAVAILABLE, ex.getStatus().getCode());
  }

  Assert.assertEquals(3, service.calls);
  Assert.assertEquals(2, beforeRetry.calls);
}
 

void execute() {
  execStub.execute(
      ExecuteRequest.newBuilder()
          .setInstanceName(instanceName)
          .setActionDigest(actionDigest)
          .setSkipCacheLookup(true)
          .build(),
      this);
  noticeFuture = service.schedule(this::printStillWaiting, 30, SECONDS);
}
 

@Test
public void cacheDownloadFailureTriggersRemoteExecution() throws Exception {
  // If downloading a cached action fails, remote execution should be tried.

  RemoteSpawnRunner runner = newSpawnRunner();

  ActionResult cachedResult = ActionResult.newBuilder().setExitCode(0).build();
  when(cache.downloadActionResult(any(ActionKey.class), /* inlineOutErr= */ eq(false)))
      .thenReturn(cachedResult);
  Exception downloadFailure =
      new BulkTransferException(new CacheNotFoundException(Digest.getDefaultInstance()));
  doThrow(downloadFailure)
      .when(cache)
      .download(eq(cachedResult), any(Path.class), any(FileOutErr.class), any());
  ActionResult execResult = ActionResult.newBuilder().setExitCode(31).build();
  ExecuteResponse succeeded = ExecuteResponse.newBuilder().setResult(execResult).build();
  when(executor.executeRemotely(any(ExecuteRequest.class))).thenReturn(succeeded);
  doNothing().when(cache).download(eq(execResult), any(Path.class), any(FileOutErr.class), any());

  Spawn spawn = newSimpleSpawn();

  SpawnExecutionContext policy = getSpawnContext(spawn);

  SpawnResult res = runner.exec(spawn, policy);
  assertThat(res.status()).isEqualTo(Status.NON_ZERO_EXIT);
  assertThat(res.exitCode()).isEqualTo(31);

  verify(executor).executeRemotely(any(ExecuteRequest.class));
}
 

@Test
public void testHumanReadableServerLogsSavedForFailingActionWithStatus() throws Exception {
  RemoteSpawnRunner runner = newSpawnRunner();
  Digest logDigest = digestUtil.computeAsUtf8("bla");
  Path logPath = logDir.getRelative(simpleActionId).getRelative("logname");
  com.google.rpc.Status timeoutStatus =
      com.google.rpc.Status.newBuilder().setCode(Code.DEADLINE_EXCEEDED.getNumber()).build();
  ExecuteResponse resp =
      ExecuteResponse.newBuilder()
          .putServerLogs(
              "logname", LogFile.newBuilder().setHumanReadable(true).setDigest(logDigest).build())
          .setStatus(timeoutStatus)
          .build();
  when(executor.executeRemotely(any(ExecuteRequest.class)))
      .thenThrow(new IOException(new ExecutionStatusException(resp.getStatus(), resp)));
  SettableFuture<Void> completed = SettableFuture.create();
  completed.set(null);
  when(cache.downloadFile(eq(logPath), eq(logDigest))).thenReturn(completed);

  Spawn spawn = newSimpleSpawn();
  SpawnExecutionContext policy = getSpawnContext(spawn);

  SpawnResult res = runner.exec(spawn, policy);
  assertThat(res.status()).isEqualTo(Status.TIMEOUT);

  verify(executor).executeRemotely(any(ExecuteRequest.class));
  verify(cache).downloadFile(eq(logPath), eq(logDigest));
}
 

@Test
public void treatFailedCachedActionAsCacheMiss_local() throws Exception {
  // Test that bazel treats failed cache action as a cache miss and attempts to execute action
  // locally

  remoteOptions.remoteLocalFallback = true;
  remoteOptions.remoteUploadLocalResults = true;

  ActionResult failedAction = ActionResult.newBuilder().setExitCode(1).build();
  when(cache.downloadActionResult(any(ActionKey.class), /* inlineOutErr= */ eq(false)))
      .thenReturn(failedAction);

  RemoteSpawnRunner runner = spy(newSpawnRunner());
  // Throw an IOException to trigger the local fallback.
  when(executor.executeRemotely(any(ExecuteRequest.class))).thenThrow(IOException.class);

  Spawn spawn = newSimpleSpawn();
  SpawnExecutionContext policy = getSpawnContext(spawn);

  SpawnResult succeeded =
      new SpawnResult.Builder()
          .setStatus(Status.SUCCESS)
          .setExitCode(0)
          .setRunnerName("test")
          .build();
  when(localRunner.exec(eq(spawn), eq(policy))).thenReturn(succeeded);

  runner.exec(spawn, policy);

  verify(localRunner).exec(eq(spawn), eq(policy));
  verify(runner)
      .execLocallyAndUpload(
          eq(spawn), eq(policy), any(), any(), any(), any(), /* uploadLocalResults= */ eq(true));
  verify(cache).upload(any(), any(), any(), any(), any(), any());
  verify(cache, never()).download(any(ActionResult.class), any(Path.class), eq(outErr), any());
}
 

@Test
public void failedLocalActionShouldNotBeUploaded() throws Exception {
  // Test that the outputs of a locally executed action that failed are not uploaded.

  remoteOptions.remoteLocalFallback = true;
  remoteOptions.remoteUploadLocalResults = true;

  RemoteSpawnRunner runner = spy(newSpawnRunner());

  // Throw an IOException to trigger the local fallback.
  when(executor.executeRemotely(any(ExecuteRequest.class))).thenThrow(IOException.class);

  Spawn spawn = newSimpleSpawn();
  SpawnExecutionContext policy = getSpawnContext(spawn);

  SpawnResult res = Mockito.mock(SpawnResult.class);
  when(res.exitCode()).thenReturn(1);
  when(res.status()).thenReturn(Status.EXECUTION_FAILED);
  when(localRunner.exec(eq(spawn), eq(policy))).thenReturn(res);

  assertThat(runner.exec(spawn, policy)).isSameInstanceAs(res);

  verify(localRunner).exec(eq(spawn), eq(policy));
  verify(runner)
      .execLocallyAndUpload(
          eq(spawn), eq(policy), any(), any(), any(), any(), /* uploadLocalResults= */ eq(true));
  verify(cache, never()).upload(any(), any(), any(), any(), any(), any());
}
 

@Test
public void cachableSpawnsShouldBeCached_localFallback() throws Exception {
  // Test that if a cachable spawn is executed locally due to the local fallback,
  // that its result is uploaded to the remote cache.

  remoteOptions.remoteAcceptCached = true;
  remoteOptions.remoteLocalFallback = true;
  remoteOptions.remoteUploadLocalResults = true;

  RemoteSpawnRunner runner = spy(newSpawnRunner());

  // Throw an IOException to trigger the local fallback.
  when(executor.executeRemotely(any(ExecuteRequest.class))).thenThrow(IOException.class);

  SpawnResult res =
      new SpawnResult.Builder()
          .setStatus(Status.SUCCESS)
          .setExitCode(0)
          .setRunnerName("test")
          .build();
  when(localRunner.exec(any(Spawn.class), any(SpawnExecutionContext.class))).thenReturn(res);

  Spawn spawn = newSimpleSpawn();
  SpawnExecutionContext policy = getSpawnContext(spawn);

  SpawnResult result = runner.exec(spawn, policy);
  assertThat(result.exitCode()).isEqualTo(0);
  assertThat(result.status()).isEqualTo(Status.SUCCESS);
  verify(localRunner).exec(eq(spawn), eq(policy));
  verify(runner)
      .execLocallyAndUpload(
          eq(spawn), eq(policy), any(), any(), any(), any(), /* uploadLocalResults= */ eq(true));
  verify(cache).upload(any(), any(), any(), any(), any(), any());
}
 

@Test
public void nonCachableSpawnsShouldNotBeCached_remote() throws Exception {
  // Test that if a spawn is marked "NO_CACHE" then it's not fetched from a remote cache.
  // It should be executed remotely, but marked non-cacheable to remote execution, so that
  // the action result is not saved in the remote cache.

  remoteOptions.remoteAcceptCached = true;
  remoteOptions.remoteLocalFallback = false;
  remoteOptions.remoteUploadLocalResults = true;
  remoteOptions.remoteResultCachePriority = 1;
  remoteOptions.remoteExecutionPriority = 2;

  RemoteSpawnRunner runner = newSpawnRunner();

  ExecuteResponse succeeded =
      ExecuteResponse.newBuilder()
          .setResult(ActionResult.newBuilder().setExitCode(0).build())
          .build();
  when(executor.executeRemotely(any(ExecuteRequest.class))).thenReturn(succeeded);

  Spawn spawn = simpleSpawnWithExecutionInfo(NO_CACHE);
  SpawnExecutionContext policy = getSpawnContext(spawn);

  runner.exec(spawn, policy);

  ArgumentCaptor<ExecuteRequest> requestCaptor = ArgumentCaptor.forClass(ExecuteRequest.class);
  verify(executor).executeRemotely(requestCaptor.capture());
  assertThat(requestCaptor.getValue().getSkipCacheLookup()).isTrue();
  assertThat(requestCaptor.getValue().getResultsCachePolicy().getPriority()).isEqualTo(1);
  assertThat(requestCaptor.getValue().getExecutionPolicy().getPriority()).isEqualTo(2);
  // TODO(olaola): verify that the uploaded action has the doNotCache set.

  verify(cache, never())
      .downloadActionResult(any(ActionKey.class), /* inlineOutErr= */ eq(false));
  verify(cache, never()).upload(any(), any(), any(), any(), any(), any());
  verifyNoMoreInteractions(localRunner);
}
 

@Override
public void execute(ExecuteRequest request, StreamObserver<Operation> responseObserver) {
  final String opName = UUID.randomUUID().toString();
  ListenableFuture<ActionResult> future =
      executorService.submit(Context.current().wrap(() -> execute(request, opName)));
  operationsCache.put(opName, future);
  // Send the first operation.
  responseObserver.onNext(Operation.newBuilder().setName(opName).build());
  // When the operation completes, send the result.
  waitExecution(opName, future, responseObserver);
}
 

private Operation executeAction(Digest actionDigest) {
  ExecuteRequest executeRequest =
      ExecuteRequest.newBuilder()
          .setInstanceName(INSTANCE_NAME)
          .setActionDigest(actionDigest)
          .setSkipCacheLookup(true)
          .build();

  ExecutionGrpc.ExecutionBlockingStub executeStub =
      ExecutionGrpc.newBlockingStub(inProcessChannel);

  return executeStub.execute(executeRequest).next();
}
 

@Test
public void printWarningIfCacheIsDown() throws Exception {
  // If we try to upload to a local cache, that is down a warning should be printed.

  remoteOptions.remoteUploadLocalResults = true;
  remoteOptions.remoteLocalFallback = true;

  Reporter reporter = new Reporter(new EventBus());
  StoredEventHandler eventHandler = new StoredEventHandler();
  reporter.addHandler(eventHandler);

  RemoteSpawnRunner runner = newSpawnRunner(reporter);
  // Trigger local fallback
  when(executor.executeRemotely(any(ExecuteRequest.class))).thenThrow(new IOException());

  Spawn spawn = newSimpleSpawn();
  SpawnExecutionContext policy = getSpawnContext(spawn);

  when(cache.downloadActionResult(any(ActionKey.class), /* inlineOutErr= */ eq(false)))
      .thenThrow(new IOException("cache down"));

  doThrow(new IOException("cache down"))
      .when(cache)
      .upload(any(), any(), any(), any(), any(), any());

  SpawnResult res =
      new SpawnResult.Builder()
          .setStatus(Status.SUCCESS)
          .setExitCode(0)
          .setRunnerName("test")
          .build();
  when(localRunner.exec(eq(spawn), eq(policy))).thenReturn(res);

  assertThat(runner.exec(spawn, policy)).isEqualTo(res);

  verify(localRunner).exec(eq(spawn), eq(policy));

  assertThat(eventHandler.getEvents()).hasSize(1);

  Event evt = eventHandler.getEvents().get(0);
  assertThat(evt.getKind()).isEqualTo(EventKind.WARNING);
  assertThat(evt.getMessage()).contains("fail");
  assertThat(evt.getMessage()).contains("upload");
}
 

private void testParamFilesAreMaterializedForFlag(String flag) throws Exception {
  ExecutionOptions executionOptions = Options.parse(ExecutionOptions.class, flag).getOptions();
  executionOptions.materializeParamFiles = true;
  RemoteSpawnRunner runner =
      new RemoteSpawnRunner(
          execRoot,
          Options.getDefaults(RemoteOptions.class),
          executionOptions,
          l -> true,
          /*cmdlineReporter=*/ null,
          "build-req-id",
          "command-id",
          cache,
          executor,
          retryService,
          digestUtil,
          logDir,
          /* filesToDownload= */ ImmutableSet.of());

  ExecuteResponse succeeded =
      ExecuteResponse.newBuilder()
          .setResult(ActionResult.newBuilder().setExitCode(0).build())
          .build();
  when(executor.executeRemotely(any(ExecuteRequest.class))).thenReturn(succeeded);

  ImmutableList<String> args = ImmutableList.of("--foo", "--bar");
  ParamFileActionInput input =
      new ParamFileActionInput(
          PathFragment.create("out/param_file"), args, ParameterFileType.UNQUOTED, ISO_8859_1);
  Spawn spawn =
      new SimpleSpawn(
          new FakeOwner("foo", "bar", "//dummy:label"),
          /*arguments=*/ ImmutableList.of(),
          /*environment=*/ ImmutableMap.of(),
          /*executionInfo=*/ ImmutableMap.of(),
          /*inputs=*/ NestedSetBuilder.create(Order.STABLE_ORDER, input),
          /*outputs=*/ ImmutableSet.<ActionInput>of(),
          ResourceSet.ZERO);
  SpawnExecutionContext policy = getSpawnContext(spawn);
  SpawnResult res = runner.exec(spawn, policy);
  assertThat(res.status()).isEqualTo(Status.SUCCESS);
  Path paramFile = execRoot.getRelative("out/param_file");
  assertThat(paramFile.exists()).isTrue();
  try (InputStream inputStream = paramFile.getInputStream()) {
    assertThat(
            new String(ByteStreams.toByteArray(inputStream), StandardCharsets.UTF_8).split("\n"))
        .asList()
        .containsExactly("--foo", "--bar");
  }
}