com.google.api.services.storage.model.StorageObject Java Examples

The following examples show how to use com.google.api.services.storage.model.StorageObject. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: GoogleCloudStorageReadChannelTest.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
@Test
public void initGeneration_hasGenerationId() throws IOException {
  StorageObject storageObject = newStorageObject(BUCKET_NAME, OBJECT_NAME);
  MockHttpTransport transport = mockTransport(jsonDataResponse(storageObject));

  List<HttpRequest> requests = new ArrayList<>();

  Storage storage = new Storage(transport, JSON_FACTORY, requests::add);

  GoogleCloudStorageReadOptions options =
      GoogleCloudStorageReadOptions.builder().setFastFailOnNotFound(false).build();

  GoogleCloudStorageReadChannel readChannel = createReadChannel(storage, options);
  // initialize metadata
  readChannel.size();
  assertThat(readChannel.generation()).isEqualTo(storageObject.getGeneration());
}
 
Example #2
Source File: GoogleStorageMetadataFeature.java    From cyberduck with GNU General Public License v3.0 6 votes vote down vote up
@Override
public void setMetadata(final Path file, final TransferStatus status) throws BackgroundException {
    if(file.isFile() || file.isPlaceholder()) {
        if(log.isDebugEnabled()) {
            log.debug(String.format("Write metadata %s for file %s", status, file));
        }
        try {
            session.getClient().objects().patch(containerService.getContainer(file).getName(), containerService.getKey(file),
                new StorageObject().setMetadata(status.getMetadata())).execute();
        }
        catch(IOException e) {
            final BackgroundException failure = new GoogleStorageExceptionMappingService().map("Failure to write attributes of {0}", e, file);
            if(file.isPlaceholder()) {
                if(failure instanceof NotfoundException) {
                    // No placeholder file may exist but we just have a common prefix
                    return;
                }
            }
            throw failure;
        }
    }
}
 
Example #3
Source File: GCSFilesSourceTest.java    From policyscanner with Apache License 2.0 6 votes vote down vote up
private void setUpGetFilesPage(List<String> objectNames) {
  // these are final classes, so use fakes instead of mocks.
  List<StorageObject> fakeItems = new ArrayList<>();
  for (int i = 0; i < objectNames.size(); ++i) {
    StorageObject fakeObject = new StorageObject().setName(objectNames.get(i));
    fakeItems.add(fakeObject);
  }

  Objects listObjects = new Objects().setItems(fakeItems);

  try {
    when(this.objectList.execute()).thenReturn(listObjects);
  } catch (IOException e) {
    fail("Failed to setup getFilesPage");
  }
}
 
Example #4
Source File: GoogleStorageAccessControlListFeature.java    From cyberduck with GNU General Public License v3.0 6 votes vote down vote up
@Override
public void setPermission(final Path file, final Acl acl) throws BackgroundException {
    try {
        if(containerService.isContainer(file)) {
            final List<BucketAccessControl> bucketAccessControls = this.toBucketAccessControl(acl);
            session.getClient().buckets().update(containerService.getContainer(file).getName(),
                new Bucket().setAcl(bucketAccessControls)).execute();
        }
        else {
            final List<ObjectAccessControl> objectAccessControls = this.toObjectAccessControl(acl);
            session.getClient().objects().update(containerService.getContainer(file).getName(), containerService.getKey(file),
                new StorageObject().setAcl(objectAccessControls)).execute();
        }
    }
    catch(IOException e) {
        final BackgroundException failure = new GoogleStorageExceptionMappingService().map("Cannot change permissions of {0}", e, file);
        if(file.isPlaceholder()) {
            if(failure instanceof NotfoundException) {
                // No placeholder file may exist but we just have a common prefix
                return;
            }
        }
        // 400 Bad Request response for buckets with uniform bucket-level access enabled
        throw failure;
    }
}
 
Example #5
Source File: GcsUploaderTests.java    From incubator-heron with Apache License 2.0 6 votes vote down vote up
@Test
public void verifyObjectBackedUpIfExists() throws IOException {
  // return an object to simulate that the topology has been uploaded before
  final StorageObject currentStorageObject = createStorageObject(topologyObjectName);
  Mockito.when(mockGcsController
      .getStorageObject(Mockito.matches(topologyObjectName)))
      .thenReturn(currentStorageObject);

  // return an object when we try to create one
  Mockito.when(mockGcsController
      .createStorageObject(Mockito.matches(topologyObjectName), Mockito.any(File.class)))
      .thenReturn(createStorageObject(topologyObjectName));

  uploader.initialize(createDefaultBuilder().build());

  uploader.uploadPackage();

  // verify that we copied the old topology before uploading the new one
  Mockito.verify(mockGcsController)
      .copyStorageObject(topologyObjectName, previousTopologyObjectName,
          currentStorageObject);
}
 
Example #6
Source File: StorageHandler.java    From tech-gallery with Apache License 2.0 6 votes vote down vote up
/**
 * Method to insert a image into the bucket of the cloud storage.
 *
 * @author <a href="mailto:[email protected]"> João Felipe de Medeiros Moreira </a>
 * @since 15/10/2015
 *
 * @param name of the image.
 * @param contentStream to be converted.
 *
 * @return the media link of the image.
 *
 * @throws IOException in case a IO problem.
 * @throws GeneralSecurityException in case a security problem.
 */
public static String saveImage(String name, InputStreamContent contentStream)
    throws IOException, GeneralSecurityException {
  logger.finest("###### Saving a image");
  StorageObject objectMetadata = new StorageObject()
      // Set the destination object name
      .setName(name)
      // Set the access control list to publicly read-only
      .setAcl(Arrays.asList(new ObjectAccessControl().setEntity(ALL_USERS).setRole(READER)));

  Storage client = getService();
  String bucketName = getBucket().getName();
  Storage.Objects.Insert insertRequest =
      client.objects().insert(bucketName, objectMetadata, contentStream);

  return insertRequest.execute().getMediaLink();
}
 
Example #7
Source File: StorageSample.java    From java-docs-samples with Apache License 2.0 6 votes vote down vote up
/**
 * Uploads data to an object in a bucket.
 *
 * @param name the name of the destination object.
 * @param contentType the MIME type of the data.
 * @param file the file to upload.
 * @param bucketName the name of the bucket to create the object in.
 */
public static void uploadFile(String name, String contentType, File file, String bucketName)
    throws IOException, GeneralSecurityException {
  InputStreamContent contentStream =
      new InputStreamContent(contentType, new FileInputStream(file));
  // Setting the length improves upload performance
  contentStream.setLength(file.length());
  StorageObject objectMetadata =
      new StorageObject()
          // Set the destination object name
          .setName(name)
          // Set the access control list to publicly read-only
          .setAcl(
              Arrays.asList(new ObjectAccessControl().setEntity("allUsers").setRole("READER")));

  // Do the insert
  Storage client = StorageFactory.getService();
  Storage.Objects.Insert insertRequest =
      client.objects().insert(bucketName, objectMetadata, contentStream);

  insertRequest.execute();
}
 
Example #8
Source File: GoogleCloudStorageTest.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
/**
 * Test GoogleCloudStorage.getItemInfo(StorageResourceId) when arguments represent an object in a
 * bucket.
 */
@Test
public void testGetItemInfoObject() throws IOException {
  StorageObject storageObject = newStorageObject(BUCKET_NAME, OBJECT_NAME);

  MockHttpTransport transport = mockTransport(jsonDataResponse(storageObject));

  GoogleCloudStorage gcs = mockedGcs(transport);

  GoogleCloudStorageItemInfo info =
      gcs.getItemInfo(new StorageResourceId(BUCKET_NAME, OBJECT_NAME));

  GoogleCloudStorageItemInfo expected =
      GoogleCloudStorageImpl.createItemInfoForStorageObject(
          new StorageResourceId(BUCKET_NAME, OBJECT_NAME), storageObject);

  assertThat(info).isEqualTo(expected);
  assertThat(trackingHttpRequestInitializer.getAllRequestStrings())
      .containsExactly(getRequestString(BUCKET_NAME, OBJECT_NAME))
      .inOrder();
}
 
Example #9
Source File: SeekableGCSStream.java    From dataflow-java with Apache License 2.0 6 votes vote down vote up
private static StorageObject uriToStorageObject(String uri) throws IOException {
  StorageObject object = new StorageObject();
  if (uri.startsWith(GCS_PREFIX)) {
    uri = uri.substring(GCS_PREFIX.length());
  } else {
    throw new IOException("Invalid GCS path (does not start with gs://): " + uri);
  }
  int slashPos = uri.indexOf("/");
  if (slashPos > 0) {
    object.setBucket(uri.substring(0, slashPos));
    object.setName(uri.substring(slashPos + 1));
    LOG.info("uriToStorageObject " + uri + "=" + object.getBucket() + ":" + object.getName());
  } else {
    throw new IOException("Invalid GCS path (does not have bucket/name form): " + uri);
  }
  return object;
}
 
Example #10
Source File: GoogleCloudStorageTest.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
@Test
public void testComposeSuccess() throws Exception {
  trackingHttpRequestInitializer = new TrackingHttpRequestInitializer();

  List<String> sources = ImmutableList.of("object1", "object2");

  StorageObject storageObject = newStorageObject(BUCKET_NAME, OBJECT_NAME);

  MockHttpTransport transport =
      mockTransport(jsonDataResponse(storageObject), jsonDataResponse(storageObject));

  GoogleCloudStorage gcs = mockedGcs(transport);

  gcs.compose(BUCKET_NAME, sources, OBJECT_NAME, CreateFileOptions.DEFAULT_CONTENT_TYPE);

  assertThat(trackingHttpRequestInitializer.getAllRequestStrings())
      .containsExactly(
          getRequestString(BUCKET_NAME, OBJECT_NAME),
          composeRequestString(BUCKET_NAME, OBJECT_NAME, 1))
      .inOrder();
}
 
Example #11
Source File: GoogleCloudStorageImpl.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
public void updateMetadata(GoogleCloudStorageItemInfo itemInfo, Map<String, byte[]> metadata)
    throws IOException {
  StorageResourceId resourceId = itemInfo.getResourceId();
  Preconditions.checkArgument(
      resourceId.isStorageObject(), "Expected full StorageObject ID, got %s", resourceId);

  StorageObject storageObject = new StorageObject().setMetadata(encodeMetadata(metadata));

  Storage.Objects.Patch patchObject =
      configureRequest(
              gcs.objects()
                  .patch(resourceId.getBucketName(), resourceId.getObjectName(), storageObject),
              resourceId.getBucketName())
          .setIfMetagenerationMatch(itemInfo.getMetaGeneration());

  patchObject.execute();
}
 
Example #12
Source File: GoogleCloudStorageMockitoTest.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
/**
 * Helper for the shared boilerplate of setting up the low-level "API objects" like
 * mockStorage.objects(), etc., that is common between test cases targeting {@code
 * GoogleCloudStorage.open(StorageResourceId)}.
 *
 * @param size {@link StorageObject} size
 * @param encoding {@link StorageObject} encoding
 */
private void setUpBasicMockBehaviorForOpeningReadChannel(long size, String encoding)
    throws IOException {
  when(mockStorage.objects()).thenReturn(mockStorageObjects);
  when(mockStorageObjects.get(eq(BUCKET_NAME), eq(OBJECT_NAME)))
      .thenReturn(mockStorageObjectsGet);
  when(mockClientRequestHelper.getRequestHeaders(eq(mockStorageObjectsGet)))
      .thenReturn(mockHeaders);
  when(mockStorageObjectsGet.execute())
      .thenReturn(
          new StorageObject()
              .setBucket(BUCKET_NAME)
              .setName(OBJECT_NAME)
              .setTimeCreated(new DateTime(11L))
              .setUpdated(new DateTime(12L))
              .setSize(BigInteger.valueOf(size))
              .setContentEncoding(encoding)
              .setGeneration(1L)
              .setMetageneration(1L));
}
 
Example #13
Source File: GoogleCloudStorageReadChannel.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
/**
 * Constructs an instance of GoogleCloudStorageReadChannel.
 *
 * @param gcs storage object instance
 * @param resourceId contains information about a specific resource
 * @param requestHelper a ClientRequestHelper used to set any extra headers
 * @param readOptions fine-grained options specifying things like retry settings, buffering, etc.
 *     Could not be null.
 * @throws IOException on IO error
 */
public GoogleCloudStorageReadChannel(
    Storage gcs,
    StorageResourceId resourceId,
    ApiErrorExtractor errorExtractor,
    ClientRequestHelper<StorageObject> requestHelper,
    @Nonnull GoogleCloudStorageReadOptions readOptions)
    throws IOException {
  this.gcs = gcs;
  this.clientRequestHelper = requestHelper;
  this.errorExtractor = errorExtractor;
  this.readOptions = readOptions;
  this.resourceId = resourceId;

  // Initialize metadata if available.
  GoogleCloudStorageItemInfo info = getInitialMetadata();

  if (info != null) {
    initMetadata(info);
  }
}
 
Example #14
Source File: GoogleCloudStorageWriteChannel.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
/**
 * Constructs an instance of GoogleCloudStorageWriteChannel.
 *
 * @param uploadThreadPool thread pool to use for running the upload operation
 * @param gcs storage object instance
 * @param requestHelper a ClientRequestHelper to set extra headers
 * @param bucketName name of the bucket to create object in
 * @param objectName name of the object to create
 * @param contentType content type
 * @param contentEncoding content encoding
 * @param kmsKeyName Name of Cloud KMS key to use to encrypt the newly created object
 * @param writeConditions conditions on which write should be allowed to continue
 * @param objectMetadata metadata to apply to the newly created object
 */
public GoogleCloudStorageWriteChannel(
    ExecutorService uploadThreadPool,
    Storage gcs,
    ClientRequestHelper<StorageObject> requestHelper,
    String bucketName,
    String objectName,
    String contentType,
    String contentEncoding,
    String kmsKeyName,
    AsyncWriteChannelOptions options,
    ObjectWriteConditions writeConditions,
    Map<String, String> objectMetadata) {
  super(uploadThreadPool, options);
  this.gcs = gcs;
  this.setClientRequestHelper(requestHelper);
  this.bucketName = bucketName;
  this.objectName = objectName;
  if (contentType != null) {
    setContentType(contentType);
  }
  this.contentEncoding = contentEncoding;
  this.kmsKeyName = kmsKeyName;
  this.writeConditions = writeConditions;
  this.metadata = objectMetadata;
}
 
Example #15
Source File: GoogleCloudStorageWriteChannel.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
@Override
public Insert createRequest(InputStreamContent inputStream) throws IOException {
  // Create object with the given name and metadata.
  StorageObject object =
      new StorageObject()
          .setContentEncoding(contentEncoding)
          .setMetadata(metadata)
          .setName(objectName);

  Insert insert = gcs.objects().insert(bucketName, object, inputStream);
  writeConditions.apply(insert);
  if (insert.getMediaHttpUploader() != null) {
    insert.getMediaHttpUploader().setDirectUploadEnabled(isDirectUploadEnabled());
    insert.getMediaHttpUploader().setProgressListener(
      new LoggingMediaHttpUploaderProgressListener(this.objectName, MIN_LOGGING_INTERVAL_MS));
  }
  insert.setName(objectName);
  if (kmsKeyName != null) {
    insert.setKmsKeyName(kmsKeyName);
  }
  return insert;
}
 
Example #16
Source File: GoogleCloudStorageReadChannelTest.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
@Test
public void metadataInitialization_eager() throws IOException {
  StorageObject object = newStorageObject(BUCKET_NAME, OBJECT_NAME);
  MockHttpTransport transport = mockTransport(jsonDataResponse(object));

  List<HttpRequest> requests = new ArrayList<>();

  Storage storage = new Storage(transport, JSON_FACTORY, requests::add);

  GoogleCloudStorageReadOptions options =
      GoogleCloudStorageReadOptions.builder().setFastFailOnNotFound(true).build();

  GoogleCloudStorageReadChannel readChannel = createReadChannel(storage, options);

  assertThat(requests).hasSize(1);
  assertThat(readChannel.size()).isEqualTo(object.getSize().longValue());
  assertThat(requests).hasSize(1);
}
 
Example #17
Source File: GoogleCloudStorageReadChannelTest.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
@Test
public void metadataInitialization_lazy() throws IOException {
  StorageObject object = newStorageObject(BUCKET_NAME, OBJECT_NAME);
  MockHttpTransport transport = mockTransport(jsonDataResponse(object));

  List<HttpRequest> requests = new ArrayList<>();

  Storage storage = new Storage(transport, JSON_FACTORY, requests::add);

  GoogleCloudStorageReadOptions options =
      GoogleCloudStorageReadOptions.builder().setFastFailOnNotFound(false).build();

  GoogleCloudStorageReadChannel readChannel = createReadChannel(storage, options);

  assertThat(requests).isEmpty();
  assertThat(readChannel.size()).isEqualTo(object.getSize().longValue());
  assertThat(requests).hasSize(1);
}
 
Example #18
Source File: GCSHelper.java    From dataflow-java with Apache License 2.0 5 votes vote down vote up
/**
 * @param name of the file we're interested in
 * @return size of the file, in bytes
 * @throws IOException
 */
public long getFileSize(String bucket, String name) throws IOException {
  Storage.Objects.Get getObject = storage.objects().get(bucket, name);
  StorageObject object = getObject.execute();
  BigInteger size = object.getSize();
  if (size.compareTo(BigInteger.valueOf(Long.MAX_VALUE)) > 0) {
    throw new RuntimeException("File size is too big for a long!");
  }
  return size.longValue();
}
 
Example #19
Source File: OnDemandLiveStateCheckerTest.java    From policyscanner with Apache License 2.0 5 votes vote down vote up
private void setUpGetFilesPage(List<String> objectNames) {
  // these are final classes, so use fakes instead of mocks.
  List<StorageObject> fakeItems = new ArrayList<>();
  for (String anObjectName : objectNames) {
    fakeItems.add(new StorageObject().setName(anObjectName));
  }

  Objects listObjects = new Objects().setItems(fakeItems);

  try {
    when(this.objectList.execute()).thenReturn(listObjects);
  } catch (IOException e) {
    fail("Failed to setup getFilesPage");
  }
}
 
Example #20
Source File: LiveStateCheckerTest.java    From policyscanner with Apache License 2.0 5 votes vote down vote up
private void setUpGetFilesPage(List<String> objectNames) {
  // these are final classes, so use fakes instead of mocks.
  List<StorageObject> fakeItems = new ArrayList<>();
  for (String anObjectName : objectNames) {
    fakeItems.add(new StorageObject().setName(anObjectName));
  }

  Objects listObjects = new Objects().setItems(fakeItems);

  try {
    when(this.objectList.execute()).thenReturn(listObjects);
  } catch (IOException e) {
    fail("Failed to setup getFilesPage");
  }
}
 
Example #21
Source File: GoogleCloudStorageTest.java    From hadoop-connectors with Apache License 2.0 5 votes vote down vote up
static StorageObject newStorageObject(String bucketName, String objectName) {
  Random r = new Random();
  return new StorageObject()
      .setBucket(bucketName)
      .setName(objectName)
      .setSize(BigInteger.valueOf(r.nextInt(Integer.MAX_VALUE)))
      .setStorageClass("standard")
      .setGeneration((long) r.nextInt(Integer.MAX_VALUE))
      .setMetageneration((long) r.nextInt(Integer.MAX_VALUE))
      .setTimeCreated(new DateTime(new Date()))
      .setUpdated(new DateTime(new Date()));
}
 
Example #22
Source File: GoogleCloudStorageImpl.java    From hadoop-connectors with Apache License 2.0 5 votes vote down vote up
/**
 * Helper for creating a Storage.Objects.Copy object ready for dispatch given a bucket and object
 * for an empty object to be created. Caller must already verify that {@code resourceId}
 * represents a StorageObject and not a bucket.
 */
private Storage.Objects.Insert prepareEmptyInsert(
    StorageResourceId resourceId, CreateObjectOptions createObjectOptions) throws IOException {
  Map<String, String> rewrittenMetadata = encodeMetadata(createObjectOptions.getMetadata());
  StorageObject object =
      new StorageObject()
          .setName(resourceId.getObjectName())
          .setMetadata(rewrittenMetadata)
          .setContentEncoding(createObjectOptions.getContentEncoding());

  // Ideally we'd use EmptyContent, but Storage requires an AbstractInputStreamContent and not
  // just an HttpContent, so we'll just use the next easiest thing.
  ByteArrayContent emptyContent =
      new ByteArrayContent(createObjectOptions.getContentType(), new byte[0]);
  Storage.Objects.Insert insertObject =
      configureRequest(
          gcs.objects().insert(resourceId.getBucketName(), object, emptyContent),
          resourceId.getBucketName());
  insertObject.setDisableGZipContent(true);
  clientRequestHelper.setDirectUploadEnabled(insertObject, true);

  if (resourceId.hasGenerationId()) {
    insertObject.setIfGenerationMatch(resourceId.getGenerationId());
  } else if (!createObjectOptions.overwriteExisting()) {
    insertObject.setIfGenerationMatch(0L);
  }
  return insertObject;
}
 
Example #23
Source File: GoogleCloudStorageImpl.java    From hadoop-connectors with Apache License 2.0 5 votes vote down vote up
/**
 * See {@link GoogleCloudStorage#listObjectInfo(String, String, String, long)} for details about
 * expected behavior.
 */
@Override
public List<GoogleCloudStorageItemInfo> listObjectInfo(
    String bucketName, String objectNamePrefix, String delimiter, long maxResults)
    throws IOException {
  logger.atFine().log(
      "listObjectInfo(%s, %s, %s, %s)", bucketName, objectNamePrefix, delimiter, maxResults);

  // Helper will handle going through pages of list results and accumulating them.
  List<StorageObject> listedObjects = new ArrayList<>();
  List<String> listedPrefixes = new ArrayList<>();
  listStorageObjectsAndPrefixes(
      bucketName,
      objectNamePrefix,
      delimiter,
      /* includeTrailingDelimiter= */ true,
      maxResults,
      listedObjects,
      listedPrefixes);

  // For the listedObjects, we simply parse each item into a GoogleCloudStorageItemInfo without
  // further work.
  List<GoogleCloudStorageItemInfo> objectInfos = new ArrayList<>(listedObjects.size());
  for (StorageObject obj : listedObjects) {
    objectInfos.add(
        createItemInfoForStorageObject(new StorageResourceId(bucketName, obj.getName()), obj));
  }

  if (listedPrefixes.isEmpty()) {
    return objectInfos;
  }

  handlePrefixes(bucketName, listedPrefixes, objectInfos);

  return objectInfos;
}
 
Example #24
Source File: GoogleCloudStorageImpl.java    From hadoop-connectors with Apache License 2.0 5 votes vote down vote up
@Override
public ListPage<GoogleCloudStorageItemInfo> listObjectInfoPage(
    String bucketName, String objectNamePrefix, String delimiter, String pageToken)
    throws IOException {
  logger.atFine().log(
      "listObjectInfoPage(%s, %s, %s, %s)", bucketName, objectNamePrefix, delimiter, pageToken);

  Storage.Objects.List listObject =
      createListRequest(
          bucketName,
          objectNamePrefix,
          delimiter,
          /* includeTrailingDelimiter= */ true,
          MAX_RESULTS_UNLIMITED);
  if (pageToken != null) {
    logger.atFine().log("listObjectInfoPage: next page %s", pageToken);
    listObject.setPageToken(pageToken);
  }

  // Helper will handle going through pages of list results and accumulating them.
  List<StorageObject> listedObjects = new ArrayList<>();
  List<String> listedPrefixes = new ArrayList<>();
  String nextPageToken =
      listStorageObjectsAndPrefixesPage(
          listObject, MAX_RESULTS_UNLIMITED, listedObjects, listedPrefixes);

  // For the listedObjects, we simply parse each item into a GoogleCloudStorageItemInfo without
  // further work.
  List<GoogleCloudStorageItemInfo> objectInfos = new ArrayList<>(listedObjects.size());
  for (StorageObject obj : listedObjects) {
    objectInfos.add(
        createItemInfoForStorageObject(new StorageResourceId(bucketName, obj.getName()), obj));
  }

  if (!listedPrefixes.isEmpty()) {
    handlePrefixes(bucketName, listedPrefixes, objectInfos);
  }

  return new ListPage<>(objectInfos, nextPageToken);
}
 
Example #25
Source File: GcsStorageService.java    From front50 with Apache License 2.0 5 votes vote down vote up
private void purgeOldVersions(String path) throws Exception {
  Storage.Objects.List listObjects = obj_api.list(bucketName).setPrefix(path).setVersions(true);

  com.google.api.services.storage.model.Objects objects;

  // Keep the 0th object on the first page (which is current).
  List<Long> generations = new ArrayList(32);
  do {
    objects = timeExecute(listTimer, listObjects);
    List<StorageObject> items = objects.getItems();
    if (items != null) {
      int n = items.size();
      while (--n >= 0) {
        generations.add(items.get(n).getGeneration());
      }
    }
    listObjects.setPageToken(objects.getNextPageToken());
  } while (objects.getNextPageToken() != null);

  for (long generation : generations) {
    if (generation == generations.get(0)) {
      continue;
    }
    log.debug("Remove {} generation {}", value("path", path), value("generation", generation));
    timeExecute(purgeTimer, obj_api.delete(bucketName, path).setGeneration(generation));
  }
}
 
Example #26
Source File: GcsUploaderTests.java    From incubator-heron with Apache License 2.0 5 votes vote down vote up
@Test
public void doNotRestorePreviousVersionIfItDoesNotExist() throws IOException {
  Mockito.when(mockGcsController
      .getStorageObject(Mockito.matches(previousTopologyObjectName)))
      .thenReturn(null);

  uploader.initialize(createDefaultBuilder().build());

  uploader.undo();

  Mockito.verify(mockGcsController, Mockito.never())
      .copyStorageObject(Mockito.anyString(), Mockito.anyString(),
          Mockito.any(StorageObject.class));
}
 
Example #27
Source File: GcsUploaderTests.java    From incubator-heron with Apache License 2.0 5 votes vote down vote up
@Test
public void restorePreviousVersionOnUndo() throws IOException {
  final StorageObject previousObject = createStorageObject(previousTopologyObjectName);
  Mockito.when(mockGcsController
      .getStorageObject(Mockito.matches(previousTopologyObjectName)))
      .thenReturn(previousObject);

  uploader.initialize(createDefaultBuilder().build());

  uploader.undo();

  // verify that we restored the previous topology
  Mockito.verify(mockGcsController)
      .copyStorageObject(previousTopologyObjectName, topologyObjectName, previousObject);
}
 
Example #28
Source File: GoogleCloudStorageImpl.java    From hadoop-connectors with Apache License 2.0 5 votes vote down vote up
/**
 * See {@link GoogleCloudStorage#getItemInfo(StorageResourceId)} for details about expected
 * behavior.
 */
@Override
public GoogleCloudStorageItemInfo getItemInfo(StorageResourceId resourceId)
    throws IOException {
  logger.atFine().log("getItemInfo(%s)", resourceId);

  // Handle ROOT case first.
  if (resourceId.isRoot()) {
    return GoogleCloudStorageItemInfo.ROOT_INFO;
  }

  GoogleCloudStorageItemInfo itemInfo = null;

  // Determine object size.
  //
  // For buckets, size is 0.
  // For objects not found, size is -1.
  // For objects that exist, size is in number of bytes.
  if (resourceId.isBucket()) {
    Bucket bucket = getBucket(resourceId.getBucketName());
    if (bucket != null) {
      itemInfo = createItemInfoForBucket(resourceId, bucket);
    }
  } else {
    StorageObject object = getObject(resourceId);
    if (object != null) {
      itemInfo = createItemInfoForStorageObject(resourceId, object);
    }
  }

  if (itemInfo == null) {
    itemInfo = GoogleCloudStorageItemInfo.createNotFound(resourceId);
  }
  logger.atFine().log("getItemInfo: %s", itemInfo);
  return itemInfo;
}
 
Example #29
Source File: GcsController.java    From incubator-heron with Apache License 2.0 5 votes vote down vote up
StorageObject getStorageObject(String storageObjectName) {
  try {
    return storage.objects().get(bucket, storageObjectName).execute();
  } catch (IOException e) {
    // ignored
  }
  return null;
}
 
Example #30
Source File: AbelanaUpload.java    From Abelana-Android with Apache License 2.0 5 votes vote down vote up
@Override
protected Void doInBackground(Void... params) {

    try {
        InputStreamContent mediaContent = new InputStreamContent( "application/octet-stream", file_to_copy);
        // Not strictly necessary, but allows optimization in the cloud.
        // mediaContent.setLength(OBJECT_SIZE);

        StorageObject objectMetadata = null;

        Storage.Objects.Insert insertObject =
                AbelanaThings.storage.objects().insert(BUCKET, objectMetadata, mediaContent);

        // If you don't provide metadata, you will have specify the object
        // name by parameter. You will probably also want to ensure that your
        // default object ACLs (a bucket property) are set appropriately:
        // https://developers.google.com/storage/docs/json_api/v1/buckets#defaultObjectAcl
        insertObject.setName( fileName );

        insertObject.getMediaHttpUploader().setDisableGZipContent(true);
        // For small files, you may wish to call setDirectUploadEnabled(true), to
        // reduce the number of HTTP requests made to the server.
        if (mediaContent.getLength() > 0 && mediaContent.getLength() <= 2 * 1000 * 1000 /* 2MB */) {
            insertObject.getMediaHttpUploader().setDirectUploadEnabled(true);
        }
        insertObject.execute();

    } catch (IOException e) {
        e.printStackTrace();
    }

    return null;
}