build.bazel.remote.execution.v2.Digest Java Examples

@Override
public ListenableFuture<Void> downloadBlob(Digest digest, OutputStream out) {
  @Nullable
  HashingOutputStream hashOut = verifyDownloads ? digestUtil.newHashingOutputStream(out) : null;
  return Futures.transformAsync(
      download(digest, hashOut != null ? hashOut : out, /* isActionCache= */ false),
      (v) -> {
        try {
          if (hashOut != null) {
            Utils.verifyBlobContents(
                digest.getHash(), DigestUtil.hashCodeToString(hashOut.hash()));
          }
          out.flush();
          return Futures.immediateFuture(null);
        } catch (IOException e) {
          return Futures.immediateFailedFuture(e);
        }
      },
      MoreExecutors.directExecutor());
}
 

@Test
public void requeueSucceedsForValidOperation() throws Exception {
  String operationName = "valid-operation";

  when(mockBackplane.getOperation(eq(operationName)))
      .thenReturn(Operation.newBuilder().setName(operationName).build());

  Action action = createAction();
  QueuedOperation queuedOperation =
      QueuedOperation.newBuilder().setAction(action).setCommand(SIMPLE_COMMAND).build();
  ByteString queuedOperationBlob = queuedOperation.toByteString();
  Digest queuedOperationDigest = DIGEST_UTIL.compute(queuedOperationBlob);
  provideBlob(queuedOperationDigest, queuedOperationBlob);

  Digest actionDigest = DIGEST_UTIL.compute(action);
  QueueEntry queueEntry =
      QueueEntry.newBuilder()
          .setExecuteEntry(
              ExecuteEntry.newBuilder()
                  .setOperationName(operationName)
                  .setSkipCacheLookup(true)
                  .setActionDigest(actionDigest))
          .setQueuedOperationDigest(queuedOperationDigest)
          .build();
  instance.requeueOperation(queueEntry).get();
}
 

BlobWriteObserver(String resourceName, SimpleBlobStore simpleBlobStore)
    throws InvalidResourceNameException {
  Digest digest = parseUploadBlobDigest(resourceName);
  this.resourceName = resourceName;
  this.size = digest.getSizeBytes();
  buffer = new RingBufferInputStream((int) Math.min(size, BLOB_BUFFER_SIZE));
  putThread =
      new Thread(
          () -> {
            try {
              simpleBlobStore.put(digest.getHash(), size, buffer);
            } catch (IOException e) {
              buffer.shutdown();
            } catch (InterruptedException e) {
              Thread.currentThread().interrupt();
            }
          });
  putThread.start();
}
 

private void addDirectory(Path dir) throws ExecException, IOException {
  Tree.Builder tree = Tree.newBuilder();
  Directory root = computeDirectory(dir, tree);
  tree.setRoot(root);

  ByteString data = tree.build().toByteString();
  Digest digest = digestUtil.compute(data.toByteArray());

  if (result != null) {
    result
        .addOutputDirectoriesBuilder()
        .setPath(dir.relativeTo(execRoot).getPathString())
        .setTreeDigest(digest);
  }

  digestToBlobs.put(digest, data);
}
 

@Test
public void nonEmptyDigestIsDelegated() throws IOException, InterruptedException {
  ByteString content = ByteString.copyFromUtf8("Hello, World");
  Digest contentDigest = DIGEST_UTIL.compute(content);
  EmptyInputStreamFactory emptyFactory =
      new EmptyInputStreamFactory(
          new InputStreamFactory() {
            @Override
            public InputStream newInput(Digest digest, long offset) throws IOException {
              if (digest.equals(contentDigest)) {
                return content.newInput();
              }
              throw new IOException("invalid");
            }
          });
  InputStream in = emptyFactory.newInput(contentDigest, /* offset=*/ 0);
  assertThat(ByteString.readFrom(in)).isEqualTo(content);
}
 

/**
 * Adds the files in {@code inputs} as nodes to {@code tree}.
 *
 * <p>This method mutates {@code tree}.
 *
 * @param inputs map of paths to files. The key determines the path at which the file should be
 *     mounted in the tree.
 * @return the number of file nodes added to {@code tree}.
 */
private static int buildFromPaths(
    SortedMap<PathFragment, Path> inputs,
    DigestUtil digestUtil,
    Map<PathFragment, DirectoryNode> tree)
    throws IOException {
  return build(
      inputs,
      tree,
      (input, path, currDir) -> {
        if (!input.isFile(Symlinks.NOFOLLOW)) {
          throw new IOException(String.format("Input '%s' is not a file.", input));
        }
        Digest d = digestUtil.compute(input);
        currDir.addChild(new FileNode(path.getBaseName(), input, d));
        return 1;
      });
}
 

@Test
public void getSuppliesResponse() {
  ActionResult result =
      ActionResult.newBuilder().setStdoutRaw(ByteString.copyFromUtf8("out")).build();
  Digest actionDigest = DIGEST_UTIL.compute(ByteString.copyFromUtf8("in"));
  serviceRegistry.addService(
      new ActionCacheImplBase() {
        @Override
        public void getActionResult(
            GetActionResultRequest request, StreamObserver<ActionResult> responseObserver) {
          if (request.getInstanceName().equals(instanceName)
              && request.getActionDigest().equals(actionDigest)) {
            responseObserver.onNext(result);
            responseObserver.onCompleted();
          } else {
            responseObserver.onError(Status.NOT_FOUND.asException());
          }
        }
      });
  assertThat(ac.get(DigestUtil.asActionKey(actionDigest))).isEqualTo(result);
}
 

void readLimitedBlob(
    Instance instance,
    Digest digest,
    long offset,
    long limit,
    StreamObserver<ReadResponse> responseObserver) {
  ServerCallStreamObserver<ReadResponse> target =
      onErrorLogReadObserver(
          format("%s(%s)", DigestUtil.toString(digest), instance.getName()),
          offset,
          (ServerCallStreamObserver<ReadResponse>) responseObserver);
  try {
    instance.getBlob(
        digest,
        offset,
        limit,
        newChunkObserver(target),
        TracingMetadataUtils.fromCurrentContext());
  } catch (Exception e) {
    target.onError(e);
  }
}
 

void readBlob(
    Instance instance,
    Digest digest,
    long offset,
    long limit,
    StreamObserver<ReadResponse> responseObserver) {
  long available = digest.getSizeBytes() - offset;
  if (available == 0) {
    responseObserver.onCompleted();
  } else if (available < 0) {
    responseObserver.onError(OUT_OF_RANGE.asException());
  } else {
    if (limit == 0) {
      limit = available;
    } else {
      limit = Math.min(available, limit);
    }
    readLimitedBlob(instance, digest, offset, limit, responseObserver);
  }
}
 

@Test
public void incompleteWriteFileIsResumed() throws IOException {
  ByteString content = ByteString.copyFromUtf8("Hello, World");
  Digest digest = DIGEST_UTIL.compute(content);

  UUID writeId = UUID.randomUUID();
  String key = fileCache.getKey(digest, false);
  Path writePath = fileCache.getPath(key).resolveSibling(key + "." + writeId);
  try (OutputStream out = Files.newOutputStream(writePath)) {
    content.substring(0, 6).writeTo(out);
  }
  Write write = fileCache.getWrite(digest, writeId, RequestMetadata.getDefaultInstance());
  AtomicBoolean notified = new AtomicBoolean(false);
  write.addListener(() -> notified.set(true), directExecutor());
  assertThat(write.getCommittedSize()).isEqualTo(6);
  try (OutputStream out = write.getOutput(1, SECONDS, () -> {})) {
    content.substring(6).writeTo(out);
  }
  assertThat(notified.get()).isTrue();
  assertThat(write.getCommittedSize()).isEqualTo(digest.getSizeBytes());
  assertThat(write.isComplete()).isTrue();
}
 

@Test
public void testHumanReadableServerLogsSavedForFailingAction() throws Exception {
  RemoteSpawnRunner runner = newSpawnRunner();
  Digest logDigest = digestUtil.computeAsUtf8("bla");
  Path logPath = logDir.getRelative(simpleActionId).getRelative("logname");
  when(executor.executeRemotely(any(ExecuteRequest.class)))
      .thenReturn(
          ExecuteResponse.newBuilder()
              .putServerLogs(
                  "logname",
                  LogFile.newBuilder().setHumanReadable(true).setDigest(logDigest).build())
              .setResult(ActionResult.newBuilder().setExitCode(31).build())
              .build());
  SettableFuture<Void> completed = SettableFuture.create();
  completed.set(null);
  when(cache.downloadFile(eq(logPath), eq(logDigest))).thenReturn(completed);

  Spawn spawn = newSimpleSpawn();
  SpawnExecutionContext policy = getSpawnContext(spawn);

  SpawnResult res = runner.exec(spawn, policy);
  assertThat(res.status()).isEqualTo(Status.NON_ZERO_EXIT);

  verify(executor).executeRemotely(any(ExecuteRequest.class));
  verify(cache).downloadFile(eq(logPath), eq(logDigest));
}
 

private void addDirectory(Path dir) throws IllegalStateException, IOException {
  Tree.Builder tree = Tree.newBuilder();
  Directory root = computeDirectory(dir, tree);
  tree.setRoot(root);

  ByteString blob = tree.build().toByteString();
  Digest digest = digestUtil.compute(blob);
  Chunker chunker = Chunker.builder().setInput(blob).build();

  if (result != null) {
    result
        .addOutputDirectoriesBuilder()
        .setPath(execRoot.relativize(dir).toString())
        .setTreeDigest(digest);
  }

  digestToChunkers.put(digest, chunker);
}
 

private Write getWrite(String resourceName)
    throws ExcessiveWriteSizeException, InstanceNotFoundException, InvalidResourceNameException {
  switch (detectResourceOperation(resourceName)) {
    case UploadBlob:
      Digest uploadBlobDigest = parseUploadBlobDigest(resourceName);
      expectedCommittedSize = uploadBlobDigest.getSizeBytes();
      return ByteStreamService.getUploadBlobWrite(
          instances.getFromUploadBlob(resourceName),
          uploadBlobDigest,
          parseUploadBlobUUID(resourceName));
    case OperationStream:
      return ByteStreamService.getOperationStreamWrite(
          instances.getFromOperationStream(resourceName), resourceName);
    case Blob:
    default:
      throw INVALID_ARGUMENT
          .withDescription("unknown resource operation for " + resourceName)
          .asRuntimeException();
  }
}
 

@Test
public void requeueRemovesRequeuers() throws InterruptedException {
  Digest actionDigest = createAction(Action.newBuilder());
  instance.execute(
      actionDigest,
      /* skipCacheLookup=*/ true,
      ExecutionPolicy.getDefaultInstance(),
      ResultsCachePolicy.getDefaultInstance(),
      RequestMetadata.getDefaultInstance(),
      (operation) -> {});
  MatchListener listener = mock(MatchListener.class);
  when(listener.onEntry(any(QueueEntry.class))).thenReturn(true);
  instance.match(Platform.getDefaultInstance(), listener);
  ArgumentCaptor<QueueEntry> queueEntryCaptor = ArgumentCaptor.forClass(QueueEntry.class);
  verify(listener, times(1)).onEntry(queueEntryCaptor.capture());
  QueueEntry queueEntry = queueEntryCaptor.getValue();
  assertThat(queueEntry).isNotNull();
  String operationName = queueEntry.getExecuteEntry().getOperationName();
  assertThat(requeuers).isNotEmpty();
  Operation queuedOperation = outstandingOperations.get(operationName);
  assertThat(instance.isQueued(queuedOperation)).isTrue();
  instance.putOperation(queuedOperation); // requeue
  assertThat(requeuers).isEmpty();
  assertThat(outstandingOperations.get(operationName)).isEqualTo(queuedOperation);
}
 

public static void checkPreconditionFailure(
    Digest actionDigest, PreconditionFailure preconditionFailure) throws StatusException {
  if (preconditionFailure.getViolationsCount() != 0) {
    throw StatusProto.toStatusException(
        Status.newBuilder()
            .setCode(Code.FAILED_PRECONDITION.getNumber())
            .setMessage(invalidActionVerboseMessage(actionDigest, preconditionFailure))
            .addDetails(Any.pack(preconditionFailure))
            .build());
  }
}
 

private void addBlobsLocation(List<Digest> digests, String name) {
  while (!backplane.isStopped()) {
    try {
      backplane.addBlobsLocation(digests, name);
      return;
    } catch (IOException e) {
      Status status = Status.fromThrowable(e);
      if (status.getCode() != Code.UNAVAILABLE && status.getCode() != Code.DEADLINE_EXCEEDED) {
        throw status.asRuntimeException();
      }
    }
  }
  throw Status.UNAVAILABLE.withDescription("backplane was stopped").asRuntimeException();
}
 

@Test(expected = StatusException.class)
public void outputFileIsDirectoryThrowsStatusExceptionOnUpload() throws Exception {
  Files.createDirectories(root.resolve("output"));
  WorkerContext context = createTestContext();
  context.uploadOutputs(
      Digest.getDefaultInstance(),
      ActionResult.newBuilder(),
      root,
      ImmutableList.of("output"),
      ImmutableList.of());
}
 

protected <T> ListenableFuture<T> expect(
    Digest digest, Parser<T> parser, Executor executor, RequestMetadata requestMetadata) {
  // FIXME find a way to make this a transform
  SettableFuture<T> future = SettableFuture.create();
  Futures.addCallback(
      getBlobFuture(digest, requestMetadata),
      new FutureCallback<ByteString>() {
        @Override
        public void onSuccess(ByteString blob) {
          try {
            future.set(parser.parseFrom(blob));
          } catch (InvalidProtocolBufferException e) {
            logger.log(
                Level.WARNING,
                format("expect parse for %s failed", DigestUtil.toString(digest)),
                e);
            future.setException(e);
          }
        }

        @Override
        public void onFailure(Throwable t) {
          logger.log(
              Level.WARNING, format("expect for %s failed", DigestUtil.toString(digest)), t);
          future.setException(t);
        }
      },
      executor);
  return future;
}
 

public Digest createScratchInput(ActionInput input, String content) throws IOException {
  Path inputFile = execRoot.getRelative(input.getExecPath());
  FileSystemUtils.createDirectoryAndParents(inputFile.getParentDirectory());
  FileSystemUtils.writeContentAsLatin1(inputFile, content);
  Digest digest = digestUtil.compute(inputFile);
  setDigest(input, digest.getHash());
  return digest;
}
 

private synchronized Write getNonEmpty(Digest digest, UUID uuid) {
  Blob blob = storage.get(digest);
  if (blob != null) {
    return new CompleteWrite(digest.getSizeBytes());
  }
  return get(BlobWriteKey.newBuilder().setDigest(digest).setIdentifier(uuid.toString()).build());
}
 

@Test
public void testHumanReadableServerLogsSavedForFailingActionWithStatus() throws Exception {
  RemoteSpawnRunner runner = newSpawnRunner();
  Digest logDigest = digestUtil.computeAsUtf8("bla");
  Path logPath = logDir.getRelative(simpleActionId).getRelative("logname");
  com.google.rpc.Status timeoutStatus =
      com.google.rpc.Status.newBuilder().setCode(Code.DEADLINE_EXCEEDED.getNumber()).build();
  ExecuteResponse resp =
      ExecuteResponse.newBuilder()
          .putServerLogs(
              "logname", LogFile.newBuilder().setHumanReadable(true).setDigest(logDigest).build())
          .setStatus(timeoutStatus)
          .build();
  when(executor.executeRemotely(any(ExecuteRequest.class)))
      .thenThrow(new IOException(new ExecutionStatusException(resp.getStatus(), resp)));
  SettableFuture<Void> completed = SettableFuture.create();
  completed.set(null);
  when(cache.downloadFile(eq(logPath), eq(logDigest))).thenReturn(completed);

  Spawn spawn = newSimpleSpawn();
  SpawnExecutionContext policy = getSpawnContext(spawn);

  SpawnResult res = runner.exec(spawn, policy);
  assertThat(res.status()).isEqualTo(Status.TIMEOUT);

  verify(executor).executeRemotely(any(ExecuteRequest.class));
  verify(cache).downloadFile(eq(logPath), eq(logDigest));
}
 

ByteString getBlob(Digest blobDigest, long offset, long count)
    throws IndexOutOfBoundsException, InterruptedException {
  if (blobDigest.getSizeBytes() == 0) {
    if (offset == 0 && count >= 0) {
      return ByteString.EMPTY;
    } else {
      throw new IndexOutOfBoundsException();
    }
  }

  Blob blob = contentAddressableStorage.get(blobDigest);

  if (blob == null) {
    return null;
  }

  if (offset < 0
      || (blob.isEmpty() && offset > 0)
      || (!blob.isEmpty() && offset >= blob.size())
      || count < 0) {
    throw new IndexOutOfBoundsException();
  }

  long endIndex = offset + count;

  return blob.getData()
      .substring((int) offset, (int) (endIndex > blob.size() ? blob.size() : endIndex));
}
 

private ListenableFuture<Void> put(
    Path path, FileNode fileNode, ImmutableList.Builder<String> inputFiles) {
  Path filePath = path.resolve(fileNode.getName());
  Digest digest = fileNode.getDigest();
  if (digest.getSizeBytes() == 0) {
    return listeningDecorator(fetchService)
        .submit(
            () -> {
              Files.createFile(filePath);
              // ignore executable
              return null;
            });
  }
  String key = fileCache.getKey(digest, fileNode.getIsExecutable());
  return transformAsync(
      fileCache.put(digest, fileNode.getIsExecutable(), fetchService),
      (fileCachePath) -> {
        checkNotNull(key);
        // we saw null entries in the built immutable list without synchronization
        synchronized (inputFiles) {
          inputFiles.add(key);
        }
        if (fileNode.getDigest().getSizeBytes() != 0) {
          try {
            Files.createLink(filePath, fileCachePath);
          } catch (IOException e) {
            return immediateFailedFuture(e);
          }
        }
        return immediateFuture(null);
      },
      fetchService);
}
 

@Override
public void get(
    Digest digest,
    long offset,
    long count,
    ServerCallStreamObserver<ByteString> blobObserver,
    RequestMetadata requestMetadata) {
  ReadRequest request =
      ReadRequest.newBuilder()
          .setResourceName(getBlobName(digest))
          .setReadOffset(offset)
          .setReadLimit(count)
          .build();
  ByteStreamGrpc.newStub(channel)
      .withInterceptors(attachMetadataInterceptor(requestMetadata))
      .read(
          request,
          new DelegateServerCallStreamObserver<ReadResponse, ByteString>(blobObserver) {
            @Override
            public void onNext(ReadResponse response) {
              blobObserver.onNext(response.getData());
            }

            @Override
            public void onError(Throwable t) {
              blobObserver.onError(t);
            }

            @Override
            public void onCompleted() {
              blobObserver.onCompleted();
            }
          });
}
 

@Test
public void uploadRelativeFileSymlinkInDirectoryAsFile() throws Exception {
  ActionResult.Builder result = ActionResult.newBuilder();
  Path dir = fs.getPath("/execroot/dir");
  dir.createDirectory();
  Path target = fs.getPath("/execroot/target");
  FileSystemUtils.writeContent(target, new byte[] {1, 2, 3, 4, 5});
  Path link = fs.getPath("/execroot/dir/link");
  link.createSymbolicLink(PathFragment.create("../target"));

  UploadManifest um =
      new UploadManifest(
          digestUtil, result, execRoot, /*uploadSymlinks=*/ false, /*allowSymlinks=*/ true);
  um.addFiles(ImmutableList.of(dir));
  Digest digest = digestUtil.compute(target);
  assertThat(um.getDigestToFile()).containsExactly(digest, link);

  Tree tree =
      Tree.newBuilder()
          .setRoot(
              Directory.newBuilder()
                  .addFiles(FileNode.newBuilder().setName("link").setDigest(digest)))
          .build();
  Digest treeDigest = digestUtil.compute(tree);

  ActionResult.Builder expectedResult = ActionResult.newBuilder();
  expectedResult.addOutputDirectoriesBuilder().setPath("dir").setTreeDigest(treeDigest);
  assertThat(result.build()).isEqualTo(expectedResult.build());
}
 

private void fetchInputs(
    Path execDir,
    Digest inputRoot,
    Map<Digest, Directory> directoriesIndex,
    OutputDirectory outputDirectory,
    ImmutableList.Builder<String> inputFiles,
    ImmutableList.Builder<Digest> inputDirectories)
    throws IOException, InterruptedException {
  Directory directory;
  if (inputRoot.getSizeBytes() == 0) {
    directory = Directory.getDefaultInstance();
  } else {
    directory = directoriesIndex.get(inputRoot);
    if (directory == null) {
      throw new IOException(
          "Directory " + DigestUtil.toString(inputRoot) + " is not in input index");
    }
  }

  getInterruptiblyOrIOException(
      allAsList(
          fileCache.putFiles(
              directory.getFilesList(), execDir, inputFiles, newDirectExecutorService())));
  for (DirectoryNode directoryNode : directory.getDirectoriesList()) {
    Digest digest = directoryNode.getDigest();
    String name = directoryNode.getName();
    OutputDirectory childOutputDirectory =
        outputDirectory != null ? outputDirectory.getChild(name) : null;
    Path dirPath = execDir.resolve(name);
    if (childOutputDirectory != null || !config.getLinkInputDirectories()) {
      Files.createDirectories(dirPath);
      fetchInputs(
          dirPath, digest, directoriesIndex, childOutputDirectory, inputFiles, inputDirectories);
    } else {
      inputDirectories.add(digest);
      linkDirectory(dirPath, digest, directoriesIndex);
    }
  }
}
 

@Test
public void testDownloadFailsOnDigestMismatch() throws Exception {
  // Test that the download fails when a blob/file has a different content hash than expected.

  ServerChannel server = null;
  try {
    server =
        testServer.start(
            new SimpleChannelInboundHandler<FullHttpRequest>() {
              @Override
              protected void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) {
                ByteBuf data = ctx.alloc().buffer();
                ByteBufUtil.writeUtf8(data, "bar");
                DefaultFullHttpResponse response =
                    new DefaultFullHttpResponse(
                        HttpVersion.HTTP_1_1, HttpResponseStatus.OK, data);
                HttpUtil.setContentLength(response, data.readableBytes());

                ctx.writeAndFlush(response).addListener(ChannelFutureListener.CLOSE);
              }
            });

    Credentials credentials = newCredentials();
    HttpCacheClient blobStore =
        createHttpBlobStore(
            server, /* timeoutSeconds= */ 1, /* remoteVerifyDownloads= */ true, credentials);
    Digest fooDigest = DIGEST_UTIL.compute("foo".getBytes(Charsets.UTF_8));
    try (OutputStream out = new ByteArrayOutputStream()) {
      IOException e =
          assertThrows(
              IOException.class, () -> getFromFuture(blobStore.downloadBlob(fooDigest, out)));
      assertThat(e).hasMessageThat().contains(fooDigest.getHash());
      assertThat(e).hasMessageThat().contains(DIGEST_UTIL.computeAsUtf8("bar").getHash());
    }
  } finally {
    testServer.stop(server);
  }
}
 

private ListenableFuture<Void> linkDirectory(
    Path execPath, Digest digest, Map<Digest, Directory> directoriesIndex) {
  return transformAsync(
      fileCache.putDirectory(digest, directoriesIndex, fetchService),
      (cachePath) -> {
        Files.createSymbolicLink(execPath, cachePath);
        return immediateFuture(null);
      },
      fetchService);
}
 

static InputStream newInput(
    String instanceName,
    Digest digest,
    ByteStreamStub bsStub,
    ListeningScheduledExecutorService retryService)
    throws IOException {
  return ByteStreamHelper.newInput(
      blobName(instanceName, digest),
      0,
      () -> bsStub.withDeadlineAfter(10, TimeUnit.SECONDS),
      Retrier.Backoff.exponential(Duration.ofSeconds(0), Duration.ofSeconds(0), 2, 0, 5),
      Retrier.DEFAULT_IS_RETRIABLE,
      retryService);
}
 

@Override
public void findMissingBlobs(
    FindMissingBlobsRequest request, StreamObserver<FindMissingBlobsResponse> responseObserver) {
  FindMissingBlobsResponse.Builder response = FindMissingBlobsResponse.newBuilder();

  for (Digest digest : request.getBlobDigestsList()) {
    if (!cache.containsKey(digest)) {
      response.addMissingBlobDigests(digest);
    }
  }
  responseObserver.onNext(response.build());
  responseObserver.onCompleted();
}