org.sonatype.nexus.blobstore.api.BlobAttributes Java Examples

@Override
protected boolean doDeleteHard(final BlobId blobId) {
  try {
    log.debug("Hard deleting blob {}", blobId);

    boolean blobDeleted = storage.delete(getConfiguredBucketName(), contentPath(blobId));
    if (blobDeleted) {
      String attributePath = attributePath(blobId);
      BlobAttributes attributes = getBlobAttributes(blobId);
      metricsStore.recordDeletion(blobId, attributes.getMetrics().getContentSize());
      storage.delete(getConfiguredBucketName(), attributePath);
      deletedBlobIndex.remove(blobId);
    }

    return blobDeleted;
  }
  finally {
    liveBlobs.invalidate(blobId);
  }
}
 

@Test
public void whenBlobsFromDifferentRepositoriesNeedUpdatingAfterIsCalledForEachRepository() throws Exception {
  BlobAttributes blobAttributes2 = mock(BlobAttributes.class);
  Properties properties = mock(Properties.class);
  Repository repository2 = mock(Repository.class);

  BlobId blobId2 = new BlobId("86e20baa-0bca-4915-a7dc-9a4f34e72322");
  when(fileBlobStore.get(blobId2, true)).thenReturn(blob);
  when(fileBlobStore.getBlobAttributes(blobId2)).thenReturn(blobAttributes2);
  when(blobAttributes2.getProperties()).thenReturn(properties);
  when(properties.getProperty(HEADER_PREFIX + REPO_NAME_HEADER)).thenReturn("maven-central2");
  when(repositoryManager.get("maven-central2")).thenReturn(repository2);
  when(repository2.getFormat()).thenReturn(mavenFormat);

  when(fileBlobStore.getBlobIdStream()).thenReturn(Stream.of(blobId, blobId2));

  configuration.setBoolean(RESTORE_BLOBS, true);
  configuration.setBoolean(UNDELETE_BLOBS, true);
  configuration.setBoolean(INTEGRITY_CHECK, false);
  underTest.configure(configuration);

  underTest.execute();

  verify(restoreBlobStrategy).after(true, repository);
  verify(restoreBlobStrategy).after(true, repository2);
}
 

private void runTest(final String assetName,
                     final HashCode assetHash,
                     final String blobName,
                     final HashCode blobHash,
                     final Supplier<Boolean> cancel,
                     final String blobId,
                     final Blob mockBlob)
{
  Asset asset = getMockAsset(assetName, assetHash);
  BlobAttributes blobAttributes = getMockBlobAttribues(blobName, blobHash.toString());
  when(storageTx.browseAssets(any(Bucket.class))).thenReturn(newHashSet(asset));
  when(blobStore.getBlobAttributes(any())).thenReturn(blobAttributes);
  when(blobStore.get(new BlobId(blobId))).thenReturn(mockBlob);

  defaultIntegrityCheckStrategy.check(repository, blobStore, cancel, CHECK_FAILED_HANDLER);

  verify(logger).info(startsWith("Checking integrity of assets"), anyString(), anyString());

  // if cancel is invoked, we'll never see the debug line
  if (!cancel.get()) {
    verify(logger).debug(startsWith("checking asset {}"), any(Asset.class));
  }
}
 

private BlobAttributes getMockBlobAttribues(final String name, final String sha1, final boolean deleted) {
  BlobAttributes blobAttributes = mock(BlobAttributes.class);

  Properties properties = new Properties();
  if (name != null) {
    properties.setProperty(HEADER_PREFIX + BLOB_NAME_HEADER, name);
  }
  when(blobAttributes.getProperties()).thenReturn(properties);

  BlobMetrics metrics = new BlobMetrics(new DateTime(), sha1, 0);
  when(blobAttributes.getMetrics()).thenReturn(metrics);

  when(blobAttributes.isDeleted()).thenReturn(deleted);

  return blobAttributes;
}
 

@Nullable
@Override
public BlobAttributes getBlobAttributes(final BlobId blobId) {
  Path blobPath = attributePath(blobId);
  try {
    FileBlobAttributes blobAttributes = new FileBlobAttributes(blobPath);
    if (!blobAttributes.load()) {
      log.warn("Attempt to access non-existent blob {} ({})", blobId, attributePath(blobId));
      return null;
    }
    else {
      return blobAttributes;
    }
  }
  catch (Exception e) {
      log.error("Unable to load BlobAttributes for blob id: {}, path: {}, exception: {}",
          blobId, blobPath, e.getMessage(), log.isDebugEnabled() ? e : null);
    return null;
  }
}
 

@Test
public void testBlobDeleted() {
  Asset asset = getMockAsset("name", TEST_HASH1);
  assets.add(asset);

  BlobAttributes blobAttributes = getMockBlobAttribues("name", "sha1", true);
  when(blobStore.getBlobAttributes(new BlobId("blob"))).thenReturn(blobAttributes);

  defaultIntegrityCheckStrategy.check(repository, blobStore, NO_CANCEL, CHECK_FAILED_HANDLER);

  verify(defaultIntegrityCheckStrategy, never()).checkAsset(any(), any());
  verify(logger).warn(BLOB_PROPERTIES_MARKED_AS_DELETED, asset.name());

  assertThat(checkFailed, is(true));
}
 

@Override
protected BlobAttributes getBlobAttributes(final GoogleAttributesLocation attributesFilePath) throws IOException {
  GoogleCloudBlobAttributes googleCloudBlobAttributes = new GoogleCloudBlobAttributes(bucket,
      attributesFilePath.getFullPath());
  googleCloudBlobAttributes.load();
  return googleCloudBlobAttributes;
}
 

private void detect(final BlobStore blobStore, final Consumer<String> handler) {
  Stream<BlobId> blobIds = blobStore.getBlobIdStream();

  blobIds.forEach(id -> {
    BlobAttributes attributes = blobStore.getBlobAttributes(id);
    if (attributes != null) {
      checkIfOrphaned(handler, id, attributes);
    }
    else{
      log.warn("Skipping cleanup for blob {} because blob properties not found", id);
    }
  });
}
 

private void checkIfOrphaned(final Consumer<String> handler, final BlobId id, final BlobAttributes attributes) {
  String repositoryName = attributes.getHeaders().get(REPOSITORY_NAME_KEY);

  if (repositoryName != null) {
    String assetName = attributes.getHeaders().get(ASSET_NAME_KEY);

    Repository repository = repositoryManager.get(repositoryName);
    if (repository == null) {
      log.debug("Blob {} considered orphaned because repository with name {} no longer exists", id.asUniqueString(),
          repositoryName);

      handler.accept(id.asUniqueString());
    }
    else {
      findAssociatedAsset(assetName, repository).ifPresent(asset -> {
        BlobRef blobRef = asset.blobRef();
        if (blobRef != null && !blobRef.getBlobId().asUniqueString().equals(id.asUniqueString())) {
          if (!attributes.isDeleted()) {
            handler.accept(id.asUniqueString());
          }
          else {
            log.debug("Blob {} in repository {} not considered orphaned because it is already marked soft-deleted",
                id.asUniqueString(), repositoryName);
          }
        }
      });
    }
  }
}
 

@Nullable
private Long getContentSizeForDeletion(final BlobId blobId) {
  return Optional.ofNullable(getFileBlobAttributes(blobId))
        .map(BlobAttributes::getMetrics)
        .map(BlobMetrics::getContentSize)
        .orElse(null);
}
 

@Override
public void setBlobAttributes(BlobId blobId, BlobAttributes blobAttributes) {
  try {
    FileBlobAttributes fileBlobAttributes = getFileBlobAttributes(blobId);
    fileBlobAttributes.updateFrom(blobAttributes);
    fileBlobAttributes.store();
  }
  catch (Exception e) {
    log.error("Unable to set BlobAttributes for blob id: {}, exception: {}",
        blobId, e.getMessage(), log.isDebugEnabled() ? e : null);
  }
}
 

private void detect(final BlobStore blobStore, final Consumer<String> handler) {
  Stream<BlobId> blobIds = blobStore.getBlobIdStream();

  blobIds.forEach(id -> {
    BlobAttributes attributes = blobStore.getBlobAttributes(id);
    if (attributes != null) {
      checkIfOrphaned(handler, id, attributes);
    }
    else{
      log.warn("Skipping cleanup for blob {} because blob properties not found", id);
    }
  });
}
 

private void checkIfOrphaned(final Consumer<String> handler, final BlobId id, final BlobAttributes attributes) {
  String repositoryName = attributes.getHeaders().get(REPOSITORY_NAME_KEY);

  if (repositoryName != null) {
    String assetName = attributes.getHeaders().get(ASSET_NAME_KEY);

    Repository repository = repositoryManager.get(repositoryName);
    if (repository == null) {
      log.debug("Blob {} considered orphaned because repository with name {} no longer exists", id.asUniqueString(),
          repositoryName);

      handler.accept(id.asUniqueString());
    }
    else {
      findAssociatedAsset(assetName, repository).ifPresent(asset -> {
        BlobRef blobRef = asset.blob().map(AssetBlob::blobRef).orElse(null);
        if (blobRef != null && !blobRef.getBlobId().asUniqueString().equals(id.asUniqueString())) {
          if (!attributes.isDeleted()) {
            handler.accept(id.asUniqueString());
          }
          else {
            log.debug("Blob {} in repository {} not considered orphaned because it is already marked soft-deleted",
                id.asUniqueString(), repositoryName);
          }
        }
      });
    }
  }
}
 

@Override
@Guarded(by = STARTED)
public boolean undelete(@Nullable final BlobStoreUsageChecker inUseChecker, final BlobId blobId,
                        final BlobAttributes attributes,
                        final boolean isDryRun)
{
  checkNotNull(attributes);
  String logPrefix = isDryRun ? dryRunPrefix.get() : "";
  Optional<String> blobName = Optional.of(attributes)
      .map(BlobAttributes::getProperties)
      .map(p -> p.getProperty(HEADER_PREFIX + BLOB_NAME_HEADER));
  if (!blobName.isPresent()) {
    log.error("Property not present: {}, for blob id: {}, at path: {}", HEADER_PREFIX + BLOB_NAME_HEADER,
        blobId, attributePathString(blobId)); // NOSONAR
    return false;
  }
  if (attributes.isDeleted() && inUseChecker != null && inUseChecker.test(this, blobId, blobName.get())) {
    String deletedReason = attributes.getDeletedReason();
    if (!isDryRun) {
      attributes.setDeleted(false);
      attributes.setDeletedReason(null);
      try {
        doUndelete(blobId, attributes);
        attributes.store();
      }
      catch (IOException e) {
        log.error("Error while un-deleting blob id: {}, deleted reason: {}, blob store: {}, blob name: {}",
            blobId, deletedReason, blobStoreConfiguration.getName(), blobName.get(), e);
      }
    }
    log.warn(
        "{}Soft-deleted blob still in use, un-deleting blob id: {}, deleted reason: {}, blob store: {}, blob name: {}",
        logPrefix, blobId, deletedReason, blobStoreConfiguration.getName(), blobName.get());
    return true;
  }
  return false;
}
 

protected String getBlobIdFromAttributeFilePath(final T attributeFilePath) {
  if (UUID_PATTERN.matcher(attributeFilePath.getFullPath()).matches()) {
    String filename = attributeFilePath.getFileName();
    return filename.substring(0, filename.length() - BLOB_ATTRIBUTE_SUFFIX.length());
  }
  try {
    BlobAttributes fileBlobAttributes = getBlobAttributes(attributeFilePath);
    return blobIdLocationResolver.fromHeaders(fileBlobAttributes.getHeaders()).asUniqueString();
  }
  catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public boolean undelete(@Nullable final BlobStoreUsageChecker inUseChecker,
                        final BlobId blobId,
                        final BlobAttributes attributes,
                        final boolean isDryRun)
{
  return members.get().stream()
      .map((BlobStore member) -> member.undelete(inUseChecker, blobId, attributes, isDryRun))
      .anyMatch((Boolean deleted) -> deleted);
}
 

@Nullable
@Override
public BlobAttributes getBlobAttributes(final BlobId blobId) {
  return locate(blobId)
      .map((BlobStore target) -> target.getBlobAttributes(blobId))
      .orElse(null);
}
 

@Override
public void updateFrom(final BlobAttributes blobAttributes) {
  headers = blobAttributes.getHeaders();
  metrics = blobAttributes.getMetrics();
  deleted = blobAttributes.isDeleted();
  deletedReason = blobAttributes.getDeletedReason();
  deletedDateTime = blobAttributes.getDeletedDateTime();
}
 

@Override
public BlobAttributes getBlobAttributes(final S3AttributesLocation attributesFilePath) throws IOException {
  S3BlobAttributes s3BlobAttributes = new S3BlobAttributes(s3, getConfiguredBucket(),
      attributesFilePath.getFullPath());
  s3BlobAttributes.load();
  return s3BlobAttributes;
}
 

/**
 * @return the {@link BlobAttributes} for the blob, or null
 * @throws BlobStoreException if an {@link IOException} occurs
 */
@Override
@Guarded(by = STARTED)
public BlobAttributes getBlobAttributes(final BlobId blobId) {
  try {
    GoogleCloudBlobAttributes blobAttributes = new GoogleCloudBlobAttributes(bucket, attributePath(blobId));
    return blobAttributes.load() ? blobAttributes : null;
  }
  catch (IOException e) {
    log.error("Unable to load GoogleCloudBlobAttributes for blob id: {}", blobId, e);
    throw new BlobStoreException(e, blobId);
  }
}
 

@Override
@Guarded(by = STARTED)
public void setBlobAttributes(final BlobId blobId, final BlobAttributes blobAttributes) {
  GoogleCloudBlobAttributes existing = (GoogleCloudBlobAttributes) getBlobAttributes(blobId);
  if (existing != null) {
    try {
      existing.updateFrom(blobAttributes);
      existing.store();
    }
    catch (IOException e) {
      log.error("Unable to set GoogleCloudBlobAttributes for blob id: {}", blobId, e);
    }
  }
}
 

@Override
public void updateFrom(final BlobAttributes blobAttributes) {
  headers = blobAttributes.getHeaders();
  metrics = blobAttributes.getMetrics();
  deleted = blobAttributes.isDeleted();
  deletedReason = blobAttributes.getDeletedReason();
}
 

@Override
public BlobAttributes getBlobAttributes(final BlobId blobId) {
  try {
    S3BlobAttributes blobAttributes = new S3BlobAttributes(s3, getConfiguredBucket(), attributePath(blobId));
    return blobAttributes.load() ? blobAttributes : null;
  }
  catch (IOException e) {
    log.error("Unable to load S3BlobAttributes for blob id: {}", blobId, e);
    return null;
  }
}
 

@Override
public void setBlobAttributes(BlobId blobId, BlobAttributes blobAttributes) {
  try {
    S3BlobAttributes s3BlobAttributes = (S3BlobAttributes) getBlobAttributes(blobId);
    s3BlobAttributes.updateFrom(blobAttributes);
    s3BlobAttributes.store();
  }
  catch (Exception e) {
    log.error("Unable to set BlobAttributes for blob id: {}, exception: {}",
        blobId, e.getMessage(), log.isDebugEnabled() ? e : null);
  }
}
 

@Nullable
@Override
public BlobAttributes getBlobAttributes(final BlobId blobId) {
  try {
    S3BlobAttributes blobAttributes = new S3BlobAttributes(s3, getConfiguredBucket(), attributePath(blobId));
    return blobAttributes.load() ? blobAttributes : null;
  }
  catch (IOException e) {
    log.error("Unable to load S3BlobAttributes for blob id: {}", blobId, e);
    return null;
  }
}
 

@Override
public void setBlobAttributes(BlobId blobId, BlobAttributes blobAttributes) {
  try {
    S3BlobAttributes s3BlobAttributes = (S3BlobAttributes) getBlobAttributes(blobId);
    s3BlobAttributes.updateFrom(blobAttributes);
    s3BlobAttributes.store();
  }
  catch (Exception e) {
    log.error("Unable to set BlobAttributes for blob id: {}, exception: {}",
        blobId, e.getMessage(), log.isDebugEnabled() ? e : null);
  }
}
 

/**
 * returns true if the checksum matches, false otherwise
 */
private boolean checkSha1(final BlobAttributes blobAttributes, final Asset asset) {
  String assetSha1 = getAssetSha1(asset);
  String blobSha1 = getBlobSha1(blobAttributes);

  if (!Objects.equals(assetSha1, blobSha1)) {
    log.error(SHA1_MISMATCH, asset.name(), assetSha1, blobSha1);
    return false;
  }

  return true;
}
 

/**
 * Get the SHA1 from the {@link BlobAttributes}
 */
protected String getBlobSha1(final BlobAttributes blobAttributes) {
  BlobMetrics metrics = blobAttributes.getMetrics();
  checkArgument(metrics != null, "Blob attributes are missing metrics");
  String blobSha1 = metrics.getSha1Hash();
  checkArgument(blobSha1 != null, "Blob metrics are missing SHA1 hash code");
  return blobSha1;
}
 

/**
 * returns true if the name matches, false otherwise
 */
private boolean checkName(final BlobAttributes blobAttributes, final Asset asset) {
  String assetName = getAssetName(asset);
  String blobName = getBlobName(blobAttributes);

  checkArgument(blobName != null, BLOB_NAME_MISSING);
  checkArgument(assetName != null, ASSET_NAME_MISSING);

  if (!Objects.equals(assetName, blobName)) {
    log.error(NAME_MISMATCH, blobName, assetName);
    return false;
  }

  return true;
}
 

@Override
protected void doUndelete(final BlobId blobId, final BlobAttributes attributes) {
  s3.setObjectTagging(untagAsDeleted(contentPath(blobId)));
  s3.setObjectTagging(untagAsDeleted(attributePath(blobId)));
  storeMetrics.recordAddition(attributes.getMetrics().getContentSize());
}