Java Code Examples for com.google.common.hash.HashCode#fromString()

public static Write newWrite(
    Channel channel,
    String instanceName,
    Digest digest,
    UUID uuid,
    RequestMetadata requestMetadata) {
  HashCode hash = HashCode.fromString(digest.getHash());
  String resourceName =
      ByteStreamUploader.uploadResourceName(instanceName, uuid, hash, digest.getSizeBytes());
  Supplier<ByteStreamBlockingStub> bsBlockingStub =
      Suppliers.memoize(
          () ->
              ByteStreamGrpc.newBlockingStub(channel)
                  .withInterceptors(attachMetadataInterceptor(requestMetadata)));
  Supplier<ByteStreamStub> bsStub =
      Suppliers.memoize(
          () ->
              ByteStreamGrpc.newStub(channel)
                  .withInterceptors(attachMetadataInterceptor(requestMetadata)));
  return new StubWriteOutputStream(
      bsBlockingStub, bsStub, resourceName, digest.getSizeBytes(), /* autoflush=*/ false);
}
 

@Test
public void getSha1DelegatesToThriftClient() throws EdenError, IOException, TException {
  EdenClient thriftClient = createMock(EdenClient.class);

  FileSystem fs = Jimfs.newFileSystem(Configuration.unix());
  Path entry = fs.getPath("LICENSE");
  HashCode hash = HashCode.fromString("2b8b815229aa8a61e483fb4ba0588b8b6c491890");
  SHA1Result sha1Result = new SHA1Result();
  sha1Result.setSha1(hash.asBytes());
  expect(thriftClient.getSHA1("/home/mbolin/src/buck", ImmutableList.of("LICENSE")))
      .andReturn(ImmutableList.of(sha1Result));
  replay(thriftClient);

  EdenClientPool pool = new EdenClientPool(thriftClient);
  Path pathToBuck = fs.getPath("/home/mbolin/src/buck");
  Files.createDirectories(pathToBuck.resolve(".eden"));
  Files.createSymbolicLink(
      PathNormalizer.toWindowsPathIfNeeded(pathToBuck.resolve(".eden").resolve("root")),
      PathNormalizer.toWindowsPathIfNeeded(pathToBuck));

  Optional<EdenMount> mount = EdenMount.createEdenMountForProjectRoot(pathToBuck, pool);
  assertTrue("Should find mount for path.", mount.isPresent());
  assertEquals(Sha1HashCode.fromHashCode(hash), mount.get().getSha1(entry));
  verify(thriftClient);
}
 

@Override
public boolean test(String password, String passwordHash) {
  requireNonNull(password, "Required non-null password");
  requireNonNull(passwordHash, "Required non-null password's hash");
  // retrieve salt from the hash
  final int passwordHashLength = ENCRYPTED_PASSWORD_BYTES_LENGTH * 2;
  if (passwordHash.length() < passwordHashLength + SALT_BYTES_LENGTH * 2) {
    return false;
  }
  final HashCode saltHash = HashCode.fromString(passwordHash.substring(passwordHashLength));
  // sha1(password + salt)
  final HashCode hash =
      Hashing.sha512()
          .hashBytes(Bytes.concat(password.getBytes(PWD_CHARSET), saltHash.asBytes()));
  // test sha1(password + salt) + salt == passwordHash
  return (hash.toString() + saltHash.toString()).equals(passwordHash);
}
 

/**
 * @param lines that were written in the same format output by {@link #execute(ExecutionContext)}.
 */
public static ImmutableSortedMap<String, HashCode> parseClassHashes(List<String> lines) {
  Map<String, HashCode> classNames = new HashMap<>();

  for (String line : lines) {
    int lastSeparator = line.lastIndexOf(CLASS_NAME_HASH_CODE_SEPARATOR);
    String key = line.substring(0, lastSeparator);
    HashCode value = HashCode.fromString(line.substring(lastSeparator + 1));
    HashCode existing = classNames.putIfAbsent(key, value);
    if (existing != null && !existing.equals(value)) {
      throw new IllegalArgumentException(
          String.format(
              "Multiple entries with same key but differing values: %1$s=%2$s and %1$s=%3$s",
              key, value, existing));
    }
  }

  return ImmutableSortedMap.copyOf(classNames, Ordering.natural());
}
 

/**
 * Retrieves the hash maps which are stored as key, value pairs within the conanmanifest file
 * @param tx
 * @param bucket
 * @param assetPath
 * @return hashcode of the file
 */
public HashCode lookupHashFromAsset(final StorageTx tx, final Bucket bucket, final String assetPath) {
  checkNotNull(tx);
  checkNotNull(bucket);
  checkNotNull(assetPath);

  AttributesMap attributes = getConanmanifestHashes(tx, bucket, assetPath);

  if(attributes != null) {
    String filename = getFilenameFromPath(assetPath);
    if (attributes.contains(filename)) {
      return HashCode.fromString((String) attributes.get(filename));
    }
  }
  return null;
}
 

/**
 * Extract checksums of asset blob if checksums are present in asset attributes.
 */
public Map<HashAlgorithm, HashCode> getChecksums(final Iterable<HashAlgorithm> hashAlgorithms)
{
  final NestedAttributesMap checksumAttributes = attributes().child(CHECKSUM);
  final Map<HashAlgorithm, HashCode> hashCodes = Maps.newHashMap();
  for (HashAlgorithm algorithm : hashAlgorithms) {
    final HashCode hashCode = HashCode.fromString(checksumAttributes.require(algorithm.name(), String.class));
    hashCodes.put(algorithm, hashCode);
  }
  return hashCodes;
}
 

/**
 * Extract checksum of asset blob if checksum is present in asset attributes.
 */
@Nullable
public HashCode getChecksum(final HashAlgorithm hashAlgorithm)
{
  String hashCode = attributes().child(CHECKSUM).get(hashAlgorithm.name(), String.class);
  if (hashCode != null) {
    return HashCode.fromString(hashCode);
  }
  return null;
}
 

@Test
public void writeIsResumable() throws Exception {
  UUID uuid = UUID.randomUUID();
  ByteString writeContent = ByteString.copyFromUtf8("written");
  Digest digest = DIGEST_UTIL.compute(writeContent);
  String instanceName = "test";
  HashCode hash = HashCode.fromString(digest.getHash());
  String resourceName =
      ByteStreamUploader.uploadResourceName(instanceName, uuid, hash, digest.getSizeBytes());

  // better test might just put a full gRPC CAS behind an in-process and validate state
  SettableFuture<ByteString> content = SettableFuture.create();
  serviceRegistry.addService(
      new ByteStreamServiceWriter(resourceName, content, (int) digest.getSizeBytes()));

  Channel channel = InProcessChannelBuilder.forName(fakeServerName).directExecutor().build();
  GrpcCAS cas = new GrpcCAS(instanceName, channel, /* uploader=*/ null, onExpirations);
  RequestMetadata requestMetadata = RequestMetadata.getDefaultInstance();
  Write initialWrite = cas.getWrite(digest, uuid, requestMetadata);
  try (OutputStream writeOut = initialWrite.getOutput(1, SECONDS, () -> {})) {
    writeContent.substring(0, 4).writeTo(writeOut);
  }
  Write finalWrite = cas.getWrite(digest, uuid, requestMetadata);
  try (OutputStream writeOut = finalWrite.getOutput(1, SECONDS, () -> {})) {
    writeContent.substring(4).writeTo(writeOut);
  }
  assertThat(content.get(1, TimeUnit.SECONDS)).isEqualTo(writeContent);
}
 

@Test
public void executableCommandIsCorrect() {
  BuildTarget target = BuildTargetFactory.newInstance("//foo:bar");
  ProjectFilesystem filesysten =
      TestProjectFilesystems.createProjectFilesystem(temporaryDir.getRoot());
  BuildRuleParams params =
      new BuildRuleParams(
          ImmutableSortedSet::of, ImmutableSortedSet::of, ImmutableSortedSet.of());
  Downloader downloader =
      (eventBus, uri, output) -> {
        return false;
      };
  ImmutableList<URI> uris = ImmutableList.of(URI.create("https://example.com"));
  HashCode sha256 =
      HashCode.fromString("f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2");

  HttpFileBinary binary =
      new HttpFileBinary(target, filesysten, params, downloader, uris, sha256, "foo.exe");

  ActionGraphBuilder graphBuilder = new TestActionGraphBuilder();
  graphBuilder.addToIndex(binary);

  Tool tool = binary.getExecutableCommand(OutputLabel.defaultLabel());

  Path expectedPath =
      filesysten.resolve(
          BuildTargetPaths.getGenPath(filesysten, target, "%s").resolve("foo.exe"));

  Assert.assertEquals(
      ImmutableList.of(expectedPath.toString()),
      tool.getCommandPrefix(graphBuilder.getSourcePathResolver()));
}
 

/**
 * Entry point for out of process rule execution. This should be run within the build root
 * directory (i.e. within the root cell's root).
 *
 * <p>Expected usage: {@code this_binary <build_root> <root_cell> <rule_hash> } where build_root
 * is the shared cell path ancestor and contains the rule_hash serialized data.
 */
public static void main(String[] args)
    throws IOException, StepFailedException, InterruptedException {
  LogManager.getLogManager().getLogger("").setLevel(Level.SEVERE);

  LOG.info(String.format("Started buck at time [%s].", new Date()));
  Thread.setDefaultUncaughtExceptionHandler(
      (thread, error) -> {
        error.printStackTrace(System.err);
        System.exit(1);
      });
  Preconditions.checkState(
      args.length == NUM_ARGS,
      "Expected %s arguments, got %s: <%s>",
      NUM_ARGS,
      args.length,
      Joiner.on(",").join(args));
  Path buildDir = Paths.get(args[0]);
  Path projectRoot = Paths.get(args[1]);
  HashCode hash = HashCode.fromString(args[2]);
  Path metadataPath = Paths.get(args[3]);
  new IsolatedBuildableBuilder(buildDir, projectRoot, metadataPath) {

    @Override
    protected Console createConsole() {
      return new Console(
          Verbosity.STANDARD_INFORMATION, System.out, System.err, Ansi.withoutTty());
    }

    @Override
    protected BuckEventBus createEventBus(Console console) {
      DefaultBuckEventBus buckEventBus =
          new DefaultBuckEventBus(new DefaultClock(), new BuildId("whatever"));
      buckEventBus.register(new ConsoleBuckEventListener(console));
      return buckEventBus;
    }
  }.build(hash);
  System.exit(0);
}
 

/** Returns a remote artifact and puts its metadata into the action input map. */
private Artifact createRemoteArtifact(
    String pathFragment, String contents, ActionInputMap inputs) {
  Path p = outputRoot.getRoot().asPath().getRelative(pathFragment);
  Artifact a = ActionsTestUtil.createArtifact(outputRoot, p);
  byte[] b = contents.getBytes(StandardCharsets.UTF_8);
  HashCode h = HashCode.fromString(DIGEST_UTIL.compute(b).getHash());
  FileArtifactValue f =
      new RemoteFileArtifactValue(h.asBytes(), b.length, /* locationIndex= */ 1, "action-id");
  inputs.putWithNoDepOwner(a, f);
  return a;
}
 

@Test
public void multipleBlobsUploadShouldWork() throws Exception {
  Context prevContext = withEmptyMetadata.attach();
  RemoteRetrier retrier =
      TestUtils.newRemoteRetrier(() -> new FixedBackoff(1, 0), (e) -> true, retryService);
  ByteStreamUploader uploader =
      new ByteStreamUploader(
          INSTANCE_NAME,
          new ReferenceCountedChannel(channel),
          null, /* timeout seconds */
          60,
          retrier);

  int numUploads = 10;
  Map<HashCode, byte[]> blobsByHash = Maps.newHashMap();
  Map<HashCode, Chunker> chunkers = Maps.newHashMapWithExpectedSize(numUploads);
  Random rand = new Random();
  for (int i = 0; i < numUploads; i++) {
    int blobSize = rand.nextInt(CHUNK_SIZE * 10) + CHUNK_SIZE;
    byte[] blob = new byte[blobSize];
    rand.nextBytes(blob);
    Chunker chunker = Chunker.builder().setInput(blob).setChunkSize(CHUNK_SIZE).build();
    HashCode hash = HashCode.fromString(DIGEST_UTIL.compute(blob).getHash());
    chunkers.put(hash, chunker);
    blobsByHash.put(hash, blob);
  }

  serviceRegistry.addService(new MaybeFailOnceUploadService(blobsByHash));

  uploader.uploadBlobs(chunkers, true);

  blockUntilInternalStateConsistent(uploader);

  withEmptyMetadata.detach(prevContext);
}
 

@Override
protected HashCode _deserialize(String value, DeserializationContext ctxt)
        throws IOException {
    return HashCode.fromString(value.toLowerCase(Locale.ENGLISH));
}
 

public void testSerialization() throws Exception
{
    HashCode input = HashCode.fromString("cafebabe12345678");
    String json = MAPPER.writeValueAsString(input);
    assertEquals("\"cafebabe12345678\"", json);
}
 

@Test
public void progressiveUploadCompletes() throws Exception {
  DigestUtil digestUtil = new DigestUtil(HashFunction.SHA256);
  ByteString content = ByteString.copyFromUtf8("Hello, World!");
  Digest digest = digestUtil.compute(content);
  HashCode hash = HashCode.fromString(digest.getHash());
  UUID uuid = UUID.randomUUID();
  String resourceName =
      ByteStreamUploader.uploadResourceName(INSTANCE_NAME, uuid, hash, content.size());

  assertThat(getBlob(digest)).isNull();

  SettableFuture<Void> partialFuture = SettableFuture.create();
  FutureWriteResponseObserver futureResponder = new FutureWriteResponseObserver();
  StreamObserver<WriteRequest> requestObserver =
      ByteStreamGrpc.newStub(inProcessChannel).write(futureResponder);
  ByteString shortContent = content.substring(0, 6);
  requestObserver.onNext(
      WriteRequest.newBuilder()
          .setWriteOffset(0)
          .setResourceName(resourceName)
          .setData(shortContent)
          .build());
  requestObserver.onError(Status.CANCELLED.asException());
  assertThat(futureResponder.isDone()).isTrue(); // should be done

  futureResponder = new FutureWriteResponseObserver();
  requestObserver = ByteStreamGrpc.newStub(inProcessChannel).write(futureResponder);
  requestObserver.onNext(
      WriteRequest.newBuilder()
          .setWriteOffset(6)
          .setResourceName(resourceName)
          .setData(content.substring(6))
          .setFinishWrite(true)
          .build());
  requestObserver.onCompleted();
  assertThat(futureResponder.get())
      .isEqualTo(WriteResponse.newBuilder().setCommittedSize(content.size()).build());

  assertThat(getBlob(digest)).isEqualTo(content);
}
 

@Override
public <E extends Exception> HashCode create(ValueCreator<E> creator) throws E {
  return HashCode.fromString(creator.createString());
}
 

@Test
public void deduplicationOfUploadsShouldWork() throws Exception {
  Context prevContext = withEmptyMetadata.attach();
  RemoteRetrier retrier =
      TestUtils.newRemoteRetrier(() -> mockBackoff, (e) -> true, retryService);
  ByteStreamUploader uploader =
      new ByteStreamUploader(
          INSTANCE_NAME,
          new ReferenceCountedChannel(channel),
          null, /* timeout seconds */
          60,
          retrier);

  byte[] blob = new byte[CHUNK_SIZE * 2 + 1];
  new Random().nextBytes(blob);

  Chunker chunker = Chunker.builder().setInput(blob).setChunkSize(CHUNK_SIZE).build();
  HashCode hash = HashCode.fromString(DIGEST_UTIL.compute(blob).getHash());

  AtomicInteger numUploads = new AtomicInteger();
  serviceRegistry.addService(new ByteStreamImplBase() {
    @Override
    public StreamObserver<WriteRequest> write(StreamObserver<WriteResponse> streamObserver) {
      numUploads.incrementAndGet();
      return new StreamObserver<WriteRequest>() {

        long nextOffset = 0;

        @Override
        public void onNext(WriteRequest writeRequest) {
          nextOffset += writeRequest.getData().size();
          boolean lastWrite = blob.length == nextOffset;
          assertThat(writeRequest.getFinishWrite()).isEqualTo(lastWrite);
        }

        @Override
        public void onError(Throwable throwable) {
          fail("onError should never be called.");
        }

        @Override
        public void onCompleted() {
          assertThat(nextOffset).isEqualTo(blob.length);

          WriteResponse response =
              WriteResponse.newBuilder().setCommittedSize(nextOffset).build();
          streamObserver.onNext(response);
          streamObserver.onCompleted();
        }
      };
    }
  });

  uploader.uploadBlob(hash, chunker, true);
  // This should not trigger an upload.
  uploader.uploadBlob(hash, chunker, false);

  assertThat(numUploads.get()).isEqualTo(1);

  // This test should not have triggered any retries.
  Mockito.verifyZeroInteractions(mockBackoff);

  blockUntilInternalStateConsistent(uploader);

  withEmptyMetadata.detach(prevContext);
}
 

@Test
public void unimplementedQueryShouldRestartUpload() throws Exception {
  Context prevContext = withEmptyMetadata.attach();
  Mockito.when(mockBackoff.getRetryAttempts()).thenReturn(0);
  RemoteRetrier retrier =
      TestUtils.newRemoteRetrier(() -> mockBackoff, (e) -> true, retryService);
  ByteStreamUploader uploader =
      new ByteStreamUploader(
          INSTANCE_NAME, new ReferenceCountedChannel(channel), null, 3, retrier);

  byte[] blob = new byte[CHUNK_SIZE * 2 + 1];
  new Random().nextBytes(blob);

  Chunker chunker = Chunker.builder().setInput(blob).setChunkSize(CHUNK_SIZE).build();
  HashCode hash = HashCode.fromString(DIGEST_UTIL.compute(blob).getHash());

  serviceRegistry.addService(
      new ByteStreamImplBase() {
        boolean expireCall = true;
        boolean sawReset = false;

        @Override
        public StreamObserver<WriteRequest> write(StreamObserver<WriteResponse> streamObserver) {
          return new StreamObserver<WriteRequest>() {
            @Override
            public void onNext(WriteRequest writeRequest) {
              if (expireCall) {
                streamObserver.onError(Status.DEADLINE_EXCEEDED.asException());
                expireCall = false;
              } else if (!sawReset && writeRequest.getWriteOffset() != 0) {
                streamObserver.onError(Status.INVALID_ARGUMENT.asException());
              } else {
                sawReset = true;
                if (writeRequest.getFinishWrite()) {
                  long committedSize =
                      writeRequest.getWriteOffset() + writeRequest.getData().size();
                  streamObserver.onNext(
                      WriteResponse.newBuilder().setCommittedSize(committedSize).build());
                  streamObserver.onCompleted();
                }
              }
            }

            @Override
            public void onError(Throwable throwable) {
              fail("onError should never be called.");
            }

            @Override
            public void onCompleted() {}
          };
        }

        @Override
        public void queryWriteStatus(
            QueryWriteStatusRequest request, StreamObserver<QueryWriteStatusResponse> response) {
          response.onError(Status.UNIMPLEMENTED.asException());
        }
      });

  uploader.uploadBlob(hash, chunker, true);

  // This test should have triggered a single retry, because it made
  // no progress.
  Mockito.verify(mockBackoff, Mockito.times(1)).nextDelayMillis();

  blockUntilInternalStateConsistent(uploader);

  withEmptyMetadata.detach(prevContext);
}
 

/**
 * @param hashString string that conforms to the contract of the return value of {@link
 *     com.google.common.hash.HashCode#toString()}.
 */
public RuleKey(String hashString) {
  this(HashCode.fromString(hashString));
}
 

/**
 * Checks that passed rule key value is valid and throws an {@link IllegalArgumentException} if it
 * is not.
 *
 * @param ruleKeyValue rule key to verify.
 */
@SuppressWarnings("CheckReturnValue")
private void verify(String ruleKeyValue) {
  HashCode.fromString(ruleKeyValue);
}