com.microsoft.azure.storage.blob.CloudBlobContainer Java Examples

The following examples show how to use com.microsoft.azure.storage.blob.CloudBlobContainer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AzureStorageUploader.java    From movie-db-java-on-azure with MIT License 8 votes vote down vote up
/**
 * Upload image file to Azure storage with specified name.
 *
 * @param file     image file object
 * @param fileName specified file name
 * @return relative path of the created image blob
 */
public String uploadToAzureStorage(ApplicationContext applicationContext, MultipartFile file, String fileName) {
    String uri = null;

    try {
        CloudStorageAccount storageAccount =
                (CloudStorageAccount) applicationContext.getBean("cloudStorageAccount");
        CloudBlobClient blobClient = storageAccount.createCloudBlobClient();

        setupContainer(blobClient, this.thumbnailImageContainer);
        CloudBlobContainer originalImageContainer = setupContainer(blobClient, this.originalImageContainer);

        if (originalImageContainer != null) {
            CloudBlockBlob blob = originalImageContainer.getBlockBlobReference(fileName);
            blob.upload(file.getInputStream(), file.getSize());

            uri = blob.getUri().getPath();
        }
    } catch (Exception e) {
        e.printStackTrace();
        logger.error("Error uploading image: " + e.getMessage());
    }

    return uri;
}
 
Example #2
Source File: AzureStorageService.java    From crate with Apache License 2.0 7 votes vote down vote up
public void writeBlob(String container, String blobName, InputStream inputStream, long blobSize,
                      boolean failIfAlreadyExists)
    throws URISyntaxException, StorageException, IOException {
    LOGGER.trace(() -> new ParameterizedMessage("writeBlob({}, stream, {})", blobName, blobSize));
    final Tuple<CloudBlobClient, Supplier<OperationContext>> client = client();
    final CloudBlobContainer blobContainer = client.v1().getContainerReference(container);
    final CloudBlockBlob blob = blobContainer.getBlockBlobReference(blobName);
    try {
        final AccessCondition accessCondition =
            failIfAlreadyExists ? AccessCondition.generateIfNotExistsCondition() : AccessCondition.generateEmptyCondition();
        blob.upload(inputStream, blobSize, accessCondition, null, client.v2().get());
    } catch (final StorageException se) {
        if (failIfAlreadyExists && se.getHttpStatusCode() == HttpURLConnection.HTTP_CONFLICT &&
            StorageErrorCodeStrings.BLOB_ALREADY_EXISTS.equals(se.getErrorCode())) {
            throw new FileAlreadyExistsException(blobName, null, se.getMessage());
        }
        throw se;
    }
    LOGGER.trace(() -> new ParameterizedMessage("writeBlob({}, stream, {}) - done", blobName, blobSize));
}
 
Example #3
Source File: PreviewStorageAzure.java    From arcusplatform with Apache License 2.0 7 votes vote down vote up
@Override
public void delete(String id) throws Exception {
	executor.execute(() -> { 		
		long startTime = System.nanoTime();

		CloudBlobContainer container = getContainer(id);
		try {
			CloudBlockBlob blob = container.getBlockBlobReference(id);
			blob.deleteIfExists();
			VIDEO_PREVIEW_AZURE_DELETE_SUCCESS.update(System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
		} catch (Exception e) {
			VIDEO_PREVIEW_AZURE_DELETE_FAIL.update(System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
			log.error("Failed to delete preview image {}", id, e);
		}
	});
	
}
 
Example #4
Source File: AzureBlobStorageTestAccount.java    From hadoop with Apache License 2.0 7 votes vote down vote up
private static CloudBlockBlob primeRootContainer(CloudBlobClient blobClient,
    String accountName, String blobName, int fileSize) throws Exception {

  // Create a container if it does not exist. The container name
  // must be lower case.
  CloudBlobContainer container = blobClient.getContainerReference("https://"
      + accountName + "/" + "$root");
  container.createIfNotExists();

  // Create a blob output stream.
  CloudBlockBlob blob = container.getBlockBlobReference(blobName);
  BlobOutputStream outputStream = blob.openOutputStream();

  outputStream.write(new byte[fileSize]);
  outputStream.close();

  // Return a reference to the block blob object.
  return blob;
}
 
Example #5
Source File: AzureStorageClient.java    From azure-kusto-java with MIT License 7 votes vote down vote up
CloudBlockBlob uploadStreamToBlob(InputStream inputStream, String blobName, String storageUri, boolean shouldCompress)
        throws IOException, URISyntaxException, StorageException {
    log.debug("uploadStreamToBlob: blobName: {}, storageUri: {}", blobName, storageUri);

    // Ensure
    Ensure.argIsNotNull(inputStream, "inputStream");
    Ensure.stringIsNotBlank(blobName, "blobName");
    Ensure.stringIsNotBlank(storageUri, "storageUri");

    CloudBlobContainer container = new CloudBlobContainer(new URI(storageUri));
    CloudBlockBlob blob = container.getBlockBlobReference(blobName);

    if (shouldCompress) {
        compressAndUploadStream(inputStream, blob);
    } else {
        uploadStream(inputStream, blob);
    }
    return blob;
}
 
Example #6
Source File: AzureStorageDriver.java    From dcos-cassandra-service with Apache License 2.0 7 votes vote down vote up
private CloudBlobContainer getCloudBlobContainer(String accountName, String accountKey, String containerName) {
  CloudBlobContainer container = null;

  if (StringUtils.isNotBlank(containerName)) {
    final String storageConnectionString = "DefaultEndpointsProtocol=https"
      + ";AccountName=" + accountName
      + ";AccountKey=" + accountKey;

    try {
      final CloudStorageAccount account = CloudStorageAccount.parse(storageConnectionString);
      CloudBlobClient serviceClient = account.createCloudBlobClient();

      container = serviceClient.getContainerReference(containerName);
      container.createIfNotExists();
    } catch (StorageException | URISyntaxException | InvalidKeyException e) {
      logger.error("Error connecting to container for account {} and container name {}", accountName, containerName, e);
    }
  }

  return container;
}
 
Example #7
Source File: AzureResource.java    From arcusplatform with Apache License 2.0 7 votes vote down vote up
@Override
public InputStream open() throws IOException {
   try {
      CloudBlobContainer blobContainer = blobClient.getContainerReference(container);
      blobContainer.createIfNotExists();
      CloudBlockBlob blockBlob = blobContainer.getBlockBlobReference(blob);
      if (blockBlob.exists()) {
         return blockBlob.openInputStream();
      }
      else {
         throw new IOException("Blob does not exist: " + container + ", " + blob);
      }
   }
   catch (Exception e) {
      throw new IOException("Unable to initialize blob: " + container + ", " + blob, e);
   }
}
 
Example #8
Source File: CloudFileTests.java    From azure-storage-android with Apache License 2.0 7 votes vote down vote up
@Test
@Category(SlowTests.class)
public void testCopyBlockBlobSas() throws Exception {
    // Create source on server.
    final CloudBlobContainer container = BlobTestHelper.getRandomContainerReference();
    try {
        container.create();
        final CloudBlockBlob source = container.getBlockBlobReference("source");

        source.getMetadata().put("Test", "value");
        final String data = "String data";
        source.uploadText(data, Constants.UTF8_CHARSET, null, null, null);

        final CloudFile destination = doCloudBlobCopy(source, data.length());

        final String copyData = destination.downloadText(Constants.UTF8_CHARSET, null, null, null);
        assertEquals(data, copyData);
    }
    finally {
        container.deleteIfExists();
    }
}
 
Example #9
Source File: AzureCloudBlobClientActions.java    From cloudbreak with Apache License 2.0 7 votes vote down vote up
private void listBlobsInDirectory(CloudBlobContainer cloudBlobContainer, String directoryName)
        throws URISyntaxException, StorageException {

    CloudBlobDirectory blobDirectory = cloudBlobContainer.getDirectoryReference(directoryName);

    for (ListBlobItem blobItem : blobDirectory.listBlobs()) {
        if (blobItem instanceof CloudBlobDirectory) {
            listBlobsInDirectory(cloudBlobContainer, ((CloudBlobDirectory) blobItem).getPrefix());
        } else if (blobItem instanceof CloudPageBlob) {
            Log.log(LOGGER, format(" Azure Adls Gen 2 Cloud Page Blob is present with Name: [%s] and with bytes of content: [%d] at URI: [%s] ",
                    ((CloudPageBlob) blobItem).getName(), ((CloudPageBlob) blobItem).getProperties().getLength(), blobItem.getUri().getPath()));
        } else if (blobItem instanceof CloudBlockBlob) {
            Log.log(LOGGER, format(" Azure Adls Gen 2 Cloud Block Blob is present with Name: [%s] and with bytes of content: [%d] at URI: [%s] ",
                    ((CloudBlockBlob) blobItem).getName(), ((CloudBlockBlob) blobItem).getProperties().getLength(), blobItem.getUri().getPath()));
        } else {
            LOGGER.error("Azure Adls Gen 2 Cloud Storage Item that is present at URI: [{}] cannot be classify as CloudBlob, CloudPageBlob and " +
                    "CloudBlockBlob. ", blobItem.getUri().getPath());
            throw new TestFailException(String.format("Azure Adls Gen 2 Cloud Storage Item that is present at URI: [%s] cannot be classify as" +
                    " CloudBlob, CloudPageBlob and CloudBlockBlob. ", blobItem.getUri().getPath()));
        }
    }
}
 
Example #10
Source File: AzureConnectionManager.java    From sunbird-lms-service with MIT License 7 votes vote down vote up
/**
 * This method will provide Azure CloudBlobContainer object or in case of error it will provide
 * null;
 *
 * @param containerName String
 * @return CloudBlobContainer or null
 */
public static CloudBlobContainer getContainer(String containerName, boolean isPublicAccess) {

  try {
    CloudBlobClient cloudBlobClient = getBlobClient();
    // Get a reference to a container , The container name must be lower case
    CloudBlobContainer container =
        cloudBlobClient.getContainerReference(containerName.toLowerCase(Locale.ENGLISH));
    // Create the container if it does not exist.
    boolean response = container.createIfNotExists();
    ProjectLogger.log("container creation done if not exist==" + response);
    // Create a permissions object.
    if (isPublicAccess) {
      BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
      // Include public access in the permissions object.
      containerPermissions.setPublicAccess(BlobContainerPublicAccessType.CONTAINER);
      // Set the permissions on the container.
      container.uploadPermissions(containerPermissions);
    }
    return container;
  } catch (Exception e) {
    ProjectLogger.log(e.getMessage(), e);
  }
  return null;
}
 
Example #11
Source File: VideoStorageAzure.java    From arcusplatform with Apache License 2.0 6 votes vote down vote up
@Override
public VideoStorageSession create(UUID recordingId, UUID cameraId, UUID accountId, UUID placeId, @Nullable UUID personId, long ttlInSeconds) throws Exception {
   long startTime = System.nanoTime();

   try {
      CloudBlobContainer container = getRandomContainer(recordingId, cameraId, placeId);
      CloudBlobDirectory dir = container.getDirectoryReference(placeId.toString());
      CloudAppendBlob blob = dir.getAppendBlobReference(recordingId.toString());

      blob.getProperties().setCacheControl("no-cache");
      blob.getProperties().setContentType("video/mp2t");

      blob.setStreamWriteSizeInBytes(4*1024*1024);
      VideoStorageSession result = new AzureStorageSession(recordingId, cameraId, accountId, placeId, ttlInSeconds, personId,  blob);

      VIDEO_STORAGE_AZURE_CREATE_SUCCESS.update(System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
      return result;
   } catch (Exception ex) {
      VIDEO_STORAGE_AZURE_CREATE_FAIL.update(System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
      throw ex;
   }
}
 
Example #12
Source File: BlobBasics.java    From storage-blob-java-getting-started with MIT License 6 votes vote down vote up
/**
 * Creates and returns a container for the sample application to use.
 *
 * @param blobClient CloudBlobClient object
 * @param containerName Name of the container to create
 * @return The newly created CloudBlobContainer object
 *
 * @throws StorageException
 * @throws RuntimeException
 * @throws IOException
 * @throws URISyntaxException
 * @throws IllegalArgumentException
 * @throws InvalidKeyException
 * @throws IllegalStateException
 */
private static CloudBlobContainer createContainer(CloudBlobClient blobClient, String containerName) throws StorageException, RuntimeException, IOException, InvalidKeyException, IllegalArgumentException, URISyntaxException, IllegalStateException {

    // Create a new container
    CloudBlobContainer container = blobClient.getContainerReference(containerName);
    try {
        if (container.createIfNotExists() == false) {
            throw new IllegalStateException(String.format("Container with name \"%s\" already exists.", containerName));
        }
    }
    catch (StorageException s) {
        if (s.getCause() instanceof java.net.ConnectException) {
            System.out.println("Caught connection exception from the client. If running with the default configuration please make sure you have started the storage emulator.");
        }
        throw s;
    }

    return container;
}
 
Example #13
Source File: AzureServiceFactoryTest.java    From sunbird-lms-service with MIT License 6 votes vote down vote up
@Before
public void addMockRules() {
  CloudStorageAccount cloudStorageAccount = mock(CloudStorageAccount.class);
  CloudBlobClient cloudBlobClient = mock(CloudBlobClient.class);
  CloudBlobContainer cloudBlobContainer = mock(CloudBlobContainer.class);

  ListBlobItem listBlobItem = mock(ListBlobItem.class);
  List<ListBlobItem> lst = new ArrayList<>();
  lst.add(listBlobItem);
  PowerMockito.mockStatic(CloudStorageAccount.class);
  try {
    doReturn(cloudStorageAccount).when(CloudStorageAccount.class, "parse", Mockito.anyString());
    doReturn(cloudBlobClient).when(cloudStorageAccount).createCloudBlobClient();
    doReturn(cloudBlobContainer).when(cloudBlobClient).getContainerReference(Mockito.anyString());
    doReturn(true).when(cloudBlobContainer).exists();
    when(cloudBlobContainer.listBlobs()).thenReturn(lst);
    when(listBlobItem.getUri()).thenReturn(new URI("http://www.google.com"));

  } catch (Exception e) {
    Assert.fail("Could not initalize mocks, underlying reason " + e.getLocalizedMessage());
  }
}
 
Example #14
Source File: AzureStorageService.java    From crate with Apache License 2.0 6 votes vote down vote up
public Set<String> children(String account, String container, BlobPath path) throws URISyntaxException, StorageException {
    final var blobsBuilder = new HashSet<String>();
    final Tuple<CloudBlobClient, Supplier<OperationContext>> client = client();
    final CloudBlobContainer blobContainer = client.v1().getContainerReference(container);
    final String keyPath = path.buildAsString();
    final EnumSet<BlobListingDetails> enumBlobListingDetails = EnumSet.of(BlobListingDetails.METADATA);

    for (ListBlobItem blobItem : blobContainer.listBlobs(keyPath, false, enumBlobListingDetails, null, client.v2().get())) {
        if (blobItem instanceof CloudBlobDirectory) {
            final URI uri = blobItem.getUri();
            LOGGER.trace(() -> new ParameterizedMessage("blob url [{}]", uri));
            // uri.getPath is of the form /container/keyPath.* and we want to strip off the /container/
            // this requires 1 + container.length() + 1, with each 1 corresponding to one of the /.
            // Lastly, we add the length of keyPath to the offset to strip this container's path.
            final String uriPath = uri.getPath();
            blobsBuilder.add(uriPath.substring(1 + container.length() + 1 + keyPath.length(), uriPath.length() - 1));
        }
    }
    return Set.copyOf(blobsBuilder);
}
 
Example #15
Source File: StorageAccountTests.java    From azure-storage-android with Apache License 2.0 6 votes vote down vote up
@Test
public void testCloudStorageAccountClientUriVerify() throws URISyntaxException, StorageException {
    StorageCredentialsAccountAndKey cred = new StorageCredentialsAccountAndKey(ACCOUNT_NAME, ACCOUNT_KEY);
    CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(cred, true);

    CloudBlobClient blobClient = cloudStorageAccount.createCloudBlobClient();
    CloudBlobContainer container = blobClient.getContainerReference("container1");
    assertEquals(cloudStorageAccount.getBlobEndpoint().toString() + "/container1", container.getUri().toString());

    CloudQueueClient queueClient = cloudStorageAccount.createCloudQueueClient();
    CloudQueue queue = queueClient.getQueueReference("queue1");
    assertEquals(cloudStorageAccount.getQueueEndpoint().toString() + "/queue1", queue.getUri().toString());

    CloudTableClient tableClient = cloudStorageAccount.createCloudTableClient();
    CloudTable table = tableClient.getTableReference("table1");
    assertEquals(cloudStorageAccount.getTableEndpoint().toString() + "/table1", table.getUri().toString());

    CloudFileClient fileClient = cloudStorageAccount.createCloudFileClient();
    CloudFileShare share = fileClient.getShareReference("share1");
    assertEquals(cloudStorageAccount.getFileEndpoint().toString() + "/share1", share.getUri().toString());
}
 
Example #16
Source File: ManageLinuxWebAppStorageAccountConnection.java    From azure-libraries-for-java with MIT License 6 votes vote down vote up
private static CloudBlobContainer setUpStorageAccount(String connectionString, String containerName) {
    try {
        CloudStorageAccount account = CloudStorageAccount.parse(connectionString);
        // Create a blob service client
        CloudBlobClient blobClient = account.createCloudBlobClient();
        CloudBlobContainer container = blobClient.getContainerReference(containerName);
        container.createIfNotExists();
        BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
        // Include public access in the permissions object
        containerPermissions.setPublicAccess(BlobContainerPublicAccessType.CONTAINER);
        // Set the permissions on the container
        container.uploadPermissions(containerPermissions);
        return container;
    } catch (StorageException | URISyntaxException | InvalidKeyException e) {
        throw new RuntimeException(e);
    }

}
 
Example #17
Source File: AzureStorageContainerListReaderTest.java    From components with Apache License 2.0 6 votes vote down vote up
@Test(expected = NoSuchElementException.class)
public void getCurrentOnNonAdvancableReader() {

    try {
        final List<CloudBlobContainer> list = new ArrayList<>();
        list.add(new CloudBlobContainer(new URI("https://fakeAccountName.blob.core.windows.net/container-1")));
        when(blobService.listContainers()).thenReturn(new Iterable<CloudBlobContainer>() {

            @Override
            public Iterator<CloudBlobContainer> iterator() {
                return new DummyCloudBlobContainerIterator(list);
            }
        });

        boolean startable = reader.start();
        assertTrue(startable);
        assertNotNull(reader.getCurrent());
        assertFalse(reader.advance());

        reader.getCurrent();
        fail("should throw NoSuchElementException");

    } catch (StorageException | URISyntaxException | InvalidKeyException | IOException e) {
        fail("should not throw " + e.getMessage());
    }
}
 
Example #18
Source File: AzureFileUtility.java    From sunbird-lms-service with MIT License 6 votes vote down vote up
/**
 * This method will remove the container from Azure Storage.
 *
 * @param containerName
 * @return boolean
 */
public static boolean deleteContainer(String containerName) {
  if (StringUtils.isBlank(containerName)) {
    ProjectLogger.log("Container name can't be null or empty");
    return false;
  }
  CloudBlobContainer container = AzureConnectionManager.getContainer(containerName, true);
  if (container == null) {
    ProjectLogger.log("Unable to get Azure contains object");
    return false;
  }
  try {
    boolean response = container.deleteIfExists();
    if (!response) {
      ProjectLogger.log("Container not found..");
    } else {
      ProjectLogger.log("Container is deleted===");
    }
    return true;
  } catch (Exception e) {
    ProjectLogger.log(e.getMessage(), e);
  }
  return false;
}
 
Example #19
Source File: TestContainerChecks.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testContainerChecksWithSas() throws Exception {
  testAccount = AzureBlobStorageTestAccount.create("",
      EnumSet.of(CreateOptions.UseSas));
  assumeNotNull(testAccount);
  CloudBlobContainer container = testAccount.getRealContainer();
  FileSystem fs = testAccount.getFileSystem();

  // The container shouldn't be there
  assertFalse(container.exists());

  // A write should just fail
  try {
    fs.createNewFile(new Path("/foo"));
    assertFalse("Should've thrown.", true);
  } catch (AzureException ex) {
  }
  assertFalse(container.exists());
}
 
Example #20
Source File: SecondaryTests.java    From azure-storage-android with Apache License 2.0 6 votes vote down vote up
@Test
public void testRetryOn304() throws StorageException, IOException, URISyntaxException {
    OperationContext operationContext = new OperationContext();
    operationContext.getRetryingEventHandler().addListener(new StorageEvent<RetryingEvent>() {
        @Override
        public void eventOccurred(RetryingEvent eventArg) {
            fail("Request should not be retried.");
        }
    });

    CloudBlobContainer container = BlobTestHelper.getRandomContainerReference();
    try {
        container.create();
        CloudBlockBlob blockBlobRef = (CloudBlockBlob) BlobTestHelper.uploadNewBlob(container, BlobType.BLOCK_BLOB,
                "originalBlob", 1024, null);
        AccessCondition accessCondition = AccessCondition.generateIfNoneMatchCondition(blockBlobRef.getProperties().getEtag());
        blockBlobRef.download(new ByteArrayOutputStream(), accessCondition, null, operationContext);
        
        fail("Download should fail with a 304.");
    } catch (StorageException ex) {
        assertEquals("The condition specified using HTTP conditional header(s) is not met.", ex.getMessage());
    } finally {
        container.deleteIfExists();
    }
}
 
Example #21
Source File: PreviewStorageAzure.java    From arcusplatform with Apache License 2.0 6 votes vote down vote up
@Override
public byte[] read(String id) throws IOException {
	long startTime = System.nanoTime();

	CloudBlobContainer container = getContainer(id);
	try {
		CloudBlockBlob blob = container.getBlockBlobReference(id);
		if (!blob.exists()) return null;
		blob.downloadAttributes();
		int imageSize = (int) blob.getProperties().getLength();
		byte[] image = new byte[imageSize];
		blob.downloadToByteArray(image, 0);
		VIDEO_PREVIEW_AZURE_DOWNLOAD_SUCCESS.update(System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
		return image;
	} catch (Exception e) {
		VIDEO_PREVIEW_AZURE_DOWNLOAD_FAIL.update(System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
		throw new IOException("Failed to read image " + id);
	}
}
 
Example #22
Source File: SnowflakeAzureClient.java    From snowflake-jdbc with Apache License 2.0 6 votes vote down vote up
/**
 * For a set of remote storage objects under a remote location and a given prefix/path
 * returns their properties wrapped in ObjectSummary objects
 *
 * @param remoteStorageLocation location, i.e. container for Azure
 * @param prefix                the prefix/path to list under
 * @return a collection of storage summary objects
 * @throws StorageProviderException Azure storage exception
 */
@Override
public StorageObjectSummaryCollection listObjects(String remoteStorageLocation, String prefix)
throws StorageProviderException
{
  StorageObjectSummaryCollection storageObjectSummaries;

  try
  {
    CloudBlobContainer container = azStorageClient.getContainerReference(remoteStorageLocation);
    Iterable<ListBlobItem> listBlobItemIterable = container.listBlobs(
        prefix,       // List the BLOBs under this prefix
        true          // List the BLOBs as a flat list, i.e. do not list directories
    );
    storageObjectSummaries = new StorageObjectSummaryCollection(listBlobItemIterable);
  }
  catch (URISyntaxException | StorageException ex)
  {
    logger.debug("Failed to list objects: {}", ex);
    throw new StorageProviderException(ex);
  }
  return storageObjectSummaries;
}
 
Example #23
Source File: AzureResource.java    From arcusplatform with Apache License 2.0 6 votes vote down vote up
@Override
public boolean isDirectory() {
   try {
      CloudBlobContainer blobContainer = blobClient.getContainerReference(this.container);

      if (blobContainer.exists()) {
         CloudBlobDirectory blockBlobDir = blobContainer.getDirectoryReference(this.blob);
         // Blob Directories don't exists unless they have something underneath
         return blockBlobDir.listBlobs().iterator().hasNext();
      }
   }
   catch (Exception e) {
      logger.debug("isDirectory failed to lookup URI - [{}]", getUri(), e);
   }
   return false;
}
 
Example #24
Source File: AzureResource.java    From arcusplatform with Apache License 2.0 6 votes vote down vote up
@Override
public boolean isFile() {
   try {
      CloudBlobContainer blobContainer = blobClient.getContainerReference(this.container);

      if (blobContainer.exists()) {
         CloudBlockBlob blockBlob = blobContainer.getBlockBlobReference(this.blob);
         return blockBlob.exists();
      }
   }
   catch (Exception e) {
      logger.debug("isFile failed to lookup URI - [{}]", getUri(), e);
   }
   return false;
}
 
Example #25
Source File: AzureFileManager.java    From SEAL-Demo with MIT License 6 votes vote down vote up
/**
 * Delete an existing file from Azure.
 * @param containerName Name of the Azure Blob Container
 * @param destinationName File name
 * @throws URISyntaxException if containerName contains incorrect Uri syntax
 * @throws InvalidKeyException if containerName contains an invalid key
 * @throws StorageException if the blob client is unable to get a container reference
 * @throws ExecutionException if the blob client is unable to get a container reference
 * @throws InterruptedException if the blob client is unable to get a container reference
 */
public static void DeleteFile(String containerName, String destinationName)
        throws URISyntaxException,
               StorageException,
               ExecutionException,
               InterruptedException {
    CloudBlobContainer container = getContainer(containerName);

    if (!container.exists()) {
        throw new IllegalArgumentException("Container does not exist");
    }

    CloudBlockBlob fileBlob = container.getBlockBlobReference(destinationName);
    fileBlob.deleteIfExists();
}
 
Example #26
Source File: AzureStorageDriver.java    From dcos-cassandra-service with Apache License 2.0 6 votes vote down vote up
private void uploadDirectory(String localLocation,
  CloudBlobContainer azureContainer,
  String containerName,
  String key,
  String keyspaceName,
  String cfName) throws IOException {

  logger.info(
    "uploadDirectory() localLocation: {}, containerName: {}, key: {}, keyspaceName: {}, cfName: {}",
    localLocation, containerName, key, keyspaceName, cfName);

  Files.walk(FileSystems.getDefault().getPath(localLocation)).forEach(filePath -> {
      File file = filePath.toFile();
      if (file.isFile()) {
        String fileKey = key + "/" + keyspaceName + "/" + cfName + "/" + file.getName();
        uploadFile(azureContainer, fileKey, file);
      }
    }
  );
}
 
Example #27
Source File: AzureStorageDriver.java    From dcos-cassandra-service with Apache License 2.0 5 votes vote down vote up
@Override
public void download(BackupRestoreContext ctx) throws IOException {

  final String accountName = ctx.getAccountId();
  final String accountKey = ctx.getSecretKey();
  final String localLocation = ctx.getLocalLocation();
  final String backupName = ctx.getName();
  final String nodeId = ctx.getNodeId();

  final String containerName = StringUtils.lowerCase(getContainerName(ctx.getExternalLocation()));
  // https://<account_name>.blob.core.windows.net/<container_name>
  final CloudBlobContainer container = getCloudBlobContainer(accountName, accountKey, containerName);

  if (container == null) {
    logger.error("Error uploading snapshots.  Unable to connect to {}, for container {}.",
      ctx.getExternalLocation(), containerName, localLocation);
    return;
  }
  String keyPrefix = String.format("%s/%s", backupName, nodeId);

  final Map<String, Long> snapshotFileKeys = getSnapshotFileKeys(container, keyPrefix);
  logger.info("Snapshot files for this node: {}", snapshotFileKeys);

  for (String fileKey : snapshotFileKeys.keySet()) {
    downloadFile(localLocation, container, fileKey, snapshotFileKeys.get(fileKey));
  }
}
 
Example #28
Source File: TestContainerChecks.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testContainerExistAfterDoesNotExist() throws Exception {
  testAccount = AzureBlobStorageTestAccount.create("",
      EnumSet.noneOf(CreateOptions.class));
  assumeNotNull(testAccount);
  CloudBlobContainer container = testAccount.getRealContainer();
  FileSystem fs = testAccount.getFileSystem();

  // Starting off with the container not there
  assertFalse(container.exists());

  // A list shouldn't create the container and will set file system store
  // state to DoesNotExist
  try {
    fs.listStatus(new Path("/"));
    assertTrue("Should've thrown.", false);
  } catch (FileNotFoundException ex) {
    assertTrue("Unexpected exception: " + ex,
        ex.getMessage().contains("does not exist."));
  }
  assertFalse(container.exists());

  // Create a container outside of the WASB FileSystem
  container.create();
  // Add a file to the container outside of the WASB FileSystem
  CloudBlockBlob blob = testAccount.getBlobReference("foo");
  BlobOutputStream outputStream = blob.openOutputStream();
  outputStream.write(new byte[10]);
  outputStream.close();

  // Make sure the file is visible
  assertTrue(fs.exists(new Path("/foo")));
  assertTrue(container.exists());
}
 
Example #29
Source File: DeleteAzureBlobStorage.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();

    if(flowFile == null) {
        return;
    }

    final long startNanos = System.nanoTime();
    final String containerName = context.getProperty(AzureStorageUtils.CONTAINER).evaluateAttributeExpressions(flowFile).getValue();
    final String blobPath = context.getProperty(BLOB).evaluateAttributeExpressions(flowFile).getValue();
    final String deleteSnapshotOptions = context.getProperty(DELETE_SNAPSHOTS_OPTION).getValue();

    try {
        CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), flowFile);
        CloudBlobContainer container = blobClient.getContainerReference(containerName);
        CloudBlob blob = container.getBlockBlobReference(blobPath);

        final OperationContext operationContext = new OperationContext();
        AzureStorageUtils.setProxy(operationContext, context);
        blob.deleteIfExists(DeleteSnapshotsOption.valueOf(deleteSnapshotOptions), null, null, operationContext);
        session.transfer(flowFile, REL_SUCCESS);

        final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
        session.getProvenanceReporter().invokeRemoteProcess(flowFile, blob.getSnapshotQualifiedUri().toString(), "Blob deleted");
    } catch ( StorageException | URISyntaxException e) {
        getLogger().error("Failed to delete the specified blob {} from Azure Storage. Routing to failure", new Object[]{blobPath}, e);
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
    }
}
 
Example #30
Source File: AzureContainerListService.java    From cyberduck with GNU General Public License v3.0 5 votes vote down vote up
@Override
public AttributedList<Path> list(final Path directory, final ListProgressListener listener) throws BackgroundException {
    ResultSegment<CloudBlobContainer> result;
    ResultContinuation token = null;
    try {
        final AttributedList<Path> containers = new AttributedList<Path>();
        do {
            final BlobRequestOptions options = new BlobRequestOptions();
            result = session.getClient().listContainersSegmented(null, ContainerListingDetails.NONE,
                    preferences.getInteger("azure.listing.chunksize"), token,
                    options, context);
            for(CloudBlobContainer container : result.getResults()) {
                final PathAttributes attributes = new PathAttributes();
                attributes.setETag(container.getProperties().getEtag());
                attributes.setModificationDate(container.getProperties().getLastModified().getTime());
                containers.add(new Path(PathNormalizer.normalize(container.getName()),
                        EnumSet.of(Path.Type.volume, Path.Type.directory), attributes));
            }
            listener.chunk(directory, containers);
            token = result.getContinuationToken();
        }
        while(result.getHasMoreResults());
        return containers;
    }
    catch(StorageException e) {
        throw new AzureExceptionMappingService().map("Listing directory {0} failed", e, directory);
    }
}