com.google.api.services.storage.model.Objects Java Examples

The following examples show how to use com.google.api.services.storage.model.Objects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: GcsUtilTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testNonExistentObjectReturnsEmptyResult() throws IOException {
  GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = pipelineOptions.getGcsUtil();

  Storage mockStorage = Mockito.mock(Storage.class);
  gcsUtil.setStorageClient(mockStorage);

  Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class);
  Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class);

  GcsPath pattern = GcsPath.fromUri("gs://testbucket/testdirectory/nonexistentfile");
  GoogleJsonResponseException expectedException =
      googleJsonResponseException(
          HttpStatusCodes.STATUS_CODE_NOT_FOUND, "It don't exist", "Nothing here to see");

  when(mockStorage.objects()).thenReturn(mockStorageObjects);
  when(mockStorageObjects.get(pattern.getBucket(), pattern.getObject()))
      .thenReturn(mockStorageGet);
  when(mockStorageGet.execute()).thenThrow(expectedException);

  assertEquals(Collections.emptyList(), gcsUtil.expand(pattern));
}
 
Example #2
Source File: GoogleCloudStorageImpl.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
public void updateMetadata(GoogleCloudStorageItemInfo itemInfo, Map<String, byte[]> metadata)
    throws IOException {
  StorageResourceId resourceId = itemInfo.getResourceId();
  Preconditions.checkArgument(
      resourceId.isStorageObject(), "Expected full StorageObject ID, got %s", resourceId);

  StorageObject storageObject = new StorageObject().setMetadata(encodeMetadata(metadata));

  Storage.Objects.Patch patchObject =
      configureRequest(
              gcs.objects()
                  .patch(resourceId.getBucketName(), resourceId.getObjectName(), storageObject),
              resourceId.getBucketName())
          .setIfMetagenerationMatch(itemInfo.getMetaGeneration());

  patchObject.execute();
}
 
Example #3
Source File: GoogleCloudStorageImpl.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
@Override
public void createEmptyObject(StorageResourceId resourceId, CreateObjectOptions options)
    throws IOException {
  Preconditions.checkArgument(
      resourceId.isStorageObject(), "Expected full StorageObject id, got %s", resourceId);

  Storage.Objects.Insert insertObject = prepareEmptyInsert(resourceId, options);
  try {
    insertObject.execute();
  } catch (IOException ioe) {
    if (canIgnoreExceptionForEmptyObject(ioe, resourceId, options)) {
      logger.atInfo().withCause(ioe).log(
          "Ignoring exception; verified object already exists with desired state.");
    } else {
      throw ioe;
    }
  }
}
 
Example #4
Source File: StorageSample.java    From java-docs-samples with Apache License 2.0 6 votes vote down vote up
/**
 * Uploads data to an object in a bucket.
 *
 * @param name the name of the destination object.
 * @param contentType the MIME type of the data.
 * @param file the file to upload.
 * @param bucketName the name of the bucket to create the object in.
 */
public static void uploadFile(String name, String contentType, File file, String bucketName)
    throws IOException, GeneralSecurityException {
  InputStreamContent contentStream =
      new InputStreamContent(contentType, new FileInputStream(file));
  // Setting the length improves upload performance
  contentStream.setLength(file.length());
  StorageObject objectMetadata =
      new StorageObject()
          // Set the destination object name
          .setName(name)
          // Set the access control list to publicly read-only
          .setAcl(
              Arrays.asList(new ObjectAccessControl().setEntity("allUsers").setRole("READER")));

  // Do the insert
  Storage client = StorageFactory.getService();
  Storage.Objects.Insert insertRequest =
      client.objects().insert(bucketName, objectMetadata, contentStream);

  insertRequest.execute();
}
 
Example #5
Source File: GcsUtilTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testMakeRewriteOps() throws IOException {
  GcsOptions gcsOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = gcsOptions.getGcsUtil();

  LinkedList<RewriteOp> rewrites =
      gcsUtil.makeRewriteOps(makeStrings("s", 1), makeStrings("d", 1));
  assertEquals(1, rewrites.size());

  RewriteOp rewrite = rewrites.pop();
  assertTrue(rewrite.getReadyToEnqueue());
  Storage.Objects.Rewrite request = rewrite.rewriteRequest;
  assertNull(request.getMaxBytesRewrittenPerCall());
  assertEquals("bucket", request.getSourceBucket());
  assertEquals("s0", request.getSourceObject());
  assertEquals("bucket", request.getDestinationBucket());
  assertEquals("d0", request.getDestinationObject());
}
 
Example #6
Source File: GcsUtilTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void testFileSizeNonBatch() throws Exception {
  GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = pipelineOptions.getGcsUtil();

  Storage mockStorage = Mockito.mock(Storage.class);
  gcsUtil.setStorageClient(mockStorage);

  Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class);
  Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class);

  when(mockStorage.objects()).thenReturn(mockStorageObjects);
  when(mockStorageObjects.get("testbucket", "testobject")).thenReturn(mockStorageGet);
  when(mockStorageGet.execute())
      .thenReturn(new StorageObject().setSize(BigInteger.valueOf(1000)));

  assertEquals(1000, gcsUtil.fileSize(GcsPath.fromComponents("testbucket", "testobject")));
}
 
Example #7
Source File: GoogleCloudStorageTest.java    From hadoop-connectors with Apache License 2.0 6 votes vote down vote up
/**
 * Test successful operation of GoogleCloudStorage.copy(4) where srcBucketName == dstBucketName.
 */
@Test
public void testCopyObjectsNormalOperationSameBucket() throws IOException {
  String dstObject = OBJECT_NAME + "-copy";
  StorageObject object = newStorageObject(BUCKET_NAME, dstObject);
  MockHttpTransport transport =
      mockTransport(jsonDataResponse(new Objects().setItems(ImmutableList.of(object))));

  GoogleCloudStorage gcs = mockedGcs(transport);

  gcs.copy(BUCKET_NAME, ImmutableList.of(OBJECT_NAME), BUCKET_NAME, ImmutableList.of(dstObject));

  assertThat(trackingHttpRequestInitializer.getAllRequestStrings())
      .containsExactly(
          copyRequestString(BUCKET_NAME, OBJECT_NAME, BUCKET_NAME, dstObject, "copyTo"))
      .inOrder();
}
 
Example #8
Source File: GcsUtil.java    From beam with Apache License 2.0 6 votes vote down vote up
/** Lists {@link Objects} given the {@code bucket}, {@code prefix}, {@code pageToken}. */
public Objects listObjects(String bucket, String prefix, @Nullable String pageToken)
    throws IOException {
  // List all objects that start with the prefix (including objects in sub-directories).
  Storage.Objects.List listObject = storageClient.objects().list(bucket);
  listObject.setMaxResults(MAX_LIST_ITEMS_PER_CALL);
  listObject.setPrefix(prefix);

  if (pageToken != null) {
    listObject.setPageToken(pageToken);
  }

  try {
    return ResilientOperation.retry(
        ResilientOperation.getGoogleRequestCallable(listObject),
        createBackOff(),
        RetryDeterminer.SOCKET_ERRORS,
        IOException.class);
  } catch (Exception e) {
    throw new IOException(
        String.format("Unable to match files in bucket %s, prefix %s.", bucket, prefix), e);
  }
}
 
Example #9
Source File: GCSFilesSourceTest.java    From policyscanner with Apache License 2.0 6 votes vote down vote up
private void setUpGetFilesPage(List<String> objectNames) {
  // these are final classes, so use fakes instead of mocks.
  List<StorageObject> fakeItems = new ArrayList<>();
  for (int i = 0; i < objectNames.size(); ++i) {
    StorageObject fakeObject = new StorageObject().setName(objectNames.get(i));
    fakeItems.add(fakeObject);
  }

  Objects listObjects = new Objects().setItems(fakeItems);

  try {
    when(this.objectList.execute()).thenReturn(listObjects);
  } catch (IOException e) {
    fail("Failed to setup getFilesPage");
  }
}
 
Example #10
Source File: GoogleCloudStorageTest.java    From hadoop-connectors with Apache License 2.0 5 votes vote down vote up
/** Test successful operation of GoogleCloudStorage.listObjectNames(3). */
@Test
public void testListObjectNamesPrefix() throws IOException {
  String prefix = "foo/bar/baz/";
  String pageToken = "pageToken_0";

  MockHttpTransport transport =
      mockTransport(
          jsonDataResponse(
              new Objects()
                  .setPrefixes(ImmutableList.of("foo/bar/baz/dir0/", "foo/bar/baz/dir1/"))
                  .setNextPageToken(pageToken)),
          jsonDataResponse(
              new Objects()
                  .setItems(
                      ImmutableList.of(
                          newStorageObject(BUCKET_NAME, "foo/bar/baz/"),
                          newStorageObject(BUCKET_NAME, "foo/bar/baz/obj0"),
                          newStorageObject(BUCKET_NAME, "foo/bar/baz/obj1")))
                  .setNextPageToken(null)));

  GoogleCloudStorage gcs = mockedGcs(transport);

  List<String> objectNames = gcs.listObjectNames(BUCKET_NAME, prefix, PATH_DELIMITER);

  assertThat(objectNames)
      .containsExactly(
          "foo/bar/baz/dir0/", "foo/bar/baz/dir1/", "foo/bar/baz/obj0", "foo/bar/baz/obj1")
      .inOrder();
  assertThat(trackingHttpRequestInitializer.getAllRequestStrings())
      .containsExactly(
          listRequestString(BUCKET_NAME, prefix, /* maxResults= */ 1024, /* pageToken= */ null),
          listRequestString(BUCKET_NAME, prefix, /* maxResults= */ 1024, pageToken))
      .inOrder();
}
 
Example #11
Source File: GoogleCloudStorageImpl.java    From hadoop-connectors with Apache License 2.0 5 votes vote down vote up
<RequestT extends StorageRequest<?>> RequestT configureRequest(
    RequestT request, String bucketName) {
  setRequesterPaysProject(request, bucketName);

  if (request instanceof Storage.Objects.Get || request instanceof Storage.Objects.Insert) {
    setEncryptionHeaders(request);
  }

  if (request instanceof Storage.Objects.Rewrite || request instanceof Storage.Objects.Copy) {
    setEncryptionHeaders(request);
    setDecryptionHeaders(request);
  }

  return request;
}
 
Example #12
Source File: GcsStorageService.java    From front50 with Apache License 2.0 5 votes vote down vote up
private void purgeOldVersions(String path) throws Exception {
  Storage.Objects.List listObjects = obj_api.list(bucketName).setPrefix(path).setVersions(true);

  com.google.api.services.storage.model.Objects objects;

  // Keep the 0th object on the first page (which is current).
  List<Long> generations = new ArrayList(32);
  do {
    objects = timeExecute(listTimer, listObjects);
    List<StorageObject> items = objects.getItems();
    if (items != null) {
      int n = items.size();
      while (--n >= 0) {
        generations.add(items.get(n).getGeneration());
      }
    }
    listObjects.setPageToken(objects.getNextPageToken());
  } while (objects.getNextPageToken() != null);

  for (long generation : generations) {
    if (generation == generations.get(0)) {
      continue;
    }
    log.debug("Remove {} generation {}", value("path", path), value("generation", generation));
    timeExecute(purgeTimer, obj_api.delete(bucketName, path).setGeneration(generation));
  }
}
 
Example #13
Source File: GoogleCloudStorageImpl.java    From hadoop-connectors with Apache License 2.0 5 votes vote down vote up
@Override
public ListPage<GoogleCloudStorageItemInfo> listObjectInfoPage(
    String bucketName, String objectNamePrefix, String delimiter, String pageToken)
    throws IOException {
  logger.atFine().log(
      "listObjectInfoPage(%s, %s, %s, %s)", bucketName, objectNamePrefix, delimiter, pageToken);

  Storage.Objects.List listObject =
      createListRequest(
          bucketName,
          objectNamePrefix,
          delimiter,
          /* includeTrailingDelimiter= */ true,
          MAX_RESULTS_UNLIMITED);
  if (pageToken != null) {
    logger.atFine().log("listObjectInfoPage: next page %s", pageToken);
    listObject.setPageToken(pageToken);
  }

  // Helper will handle going through pages of list results and accumulating them.
  List<StorageObject> listedObjects = new ArrayList<>();
  List<String> listedPrefixes = new ArrayList<>();
  String nextPageToken =
      listStorageObjectsAndPrefixesPage(
          listObject, MAX_RESULTS_UNLIMITED, listedObjects, listedPrefixes);

  // For the listedObjects, we simply parse each item into a GoogleCloudStorageItemInfo without
  // further work.
  List<GoogleCloudStorageItemInfo> objectInfos = new ArrayList<>(listedObjects.size());
  for (StorageObject obj : listedObjects) {
    objectInfos.add(
        createItemInfoForStorageObject(new StorageResourceId(bucketName, obj.getName()), obj));
  }

  if (!listedPrefixes.isEmpty()) {
    handlePrefixes(bucketName, listedPrefixes, objectInfos);
  }

  return new ListPage<>(objectInfos, nextPageToken);
}
 
Example #14
Source File: GoogleCloudStorageTest.java    From hadoop-connectors with Apache License 2.0 5 votes vote down vote up
/** Test GoogleCloudStorage.listObjectNames(3) with maxResults set. */
@Test
public void testListObjectNamesPrefixLimited() throws IOException {
  String prefix = "foo/bar/baz/";
  int maxResults = 3;

  MockHttpTransport transport =
      mockTransport(
          jsonDataResponse(
              new Objects()
                  .setPrefixes(ImmutableList.of("foo/bar/baz/dir0/", "foo/bar/baz/dir1/"))
                  .setItems(
                      ImmutableList.of(
                          newStorageObject(BUCKET_NAME, "foo/bar/baz/"),
                          newStorageObject(BUCKET_NAME, "foo/bar/baz/obj0")))
                  .setNextPageToken("pageToken0")));

  GoogleCloudStorage gcs = mockedGcs(transport);

  List<String> objectNames = gcs.listObjectNames(BUCKET_NAME, prefix, PATH_DELIMITER, maxResults);

  assertThat(objectNames)
      .containsExactly("foo/bar/baz/dir0/", "foo/bar/baz/dir1/", "foo/bar/baz/obj0")
      .inOrder();
  assertThat(trackingHttpRequestInitializer.getAllRequestStrings())
      .containsExactly(
          listRequestString(BUCKET_NAME, prefix, maxResults + 1, /* pageToken= */ null))
      .inOrder();
}
 
Example #15
Source File: GoogleCloudStorageImpl.java    From hadoop-connectors with Apache License 2.0 5 votes vote down vote up
private Storage.Objects.List createListRequest(
    String bucketName,
    String objectNamePrefix,
    String delimiter,
    boolean includeTrailingDelimiter,
    long maxResults)
    throws IOException {
  logger.atFine().log(
      "createListRequest(%s, %s, %s, %s, %d)",
      bucketName, objectNamePrefix, delimiter, includeTrailingDelimiter, maxResults);
  checkArgument(!Strings.isNullOrEmpty(bucketName), "bucketName must not be null or empty");

  Storage.Objects.List listObject = configureRequest(gcs.objects().list(bucketName), bucketName);

  // Set delimiter if supplied.
  if (delimiter != null) {
    listObject.setDelimiter(delimiter);
    listObject.setIncludeTrailingDelimiter(includeTrailingDelimiter);
  }

  // Set number of items to retrieve per call.
  if (maxResults <= 0 || maxResults + 1 >= storageOptions.getMaxListItemsPerCall()) {
    listObject.setMaxResults(storageOptions.getMaxListItemsPerCall());
  } else {
    // We add one in case we filter out objectNamePrefix.
    listObject.setMaxResults(maxResults + 1);
  }

  // Set prefix if supplied.
  if (!Strings.isNullOrEmpty(objectNamePrefix)) {
    listObject.setPrefix(objectNamePrefix);
  }

  return listObject;
}
 
Example #16
Source File: GoogleCloudStorageTest.java    From hadoop-connectors with Apache License 2.0 5 votes vote down vote up
@Test
public void testListObjectInfoBasic() throws IOException {
  String prefix = "foo/bar/baz/";

  StorageObject object1 = newStorageObject(BUCKET_NAME, "foo/bar/baz/obj0");
  StorageObject object2 = newStorageObject(BUCKET_NAME, "foo/bar/baz/obj1");

  List<StorageObject> objects =
      ImmutableList.of(newStorageObject(BUCKET_NAME, "foo/bar/baz/"), object1, object2);

  MockHttpTransport transport =
      mockTransport(jsonDataResponse(new Objects().setItems(objects).setNextPageToken(null)));

  GoogleCloudStorage gcs = mockedGcs(transport);

  List<GoogleCloudStorageItemInfo> objectInfos =
      gcs.listObjectInfo(BUCKET_NAME, prefix, PATH_DELIMITER);

  // The item exactly matching the input prefix will be discarded.
  assertThat(objectInfos)
      .containsExactly(
          createItemInfoForStorageObject(
              new StorageResourceId(BUCKET_NAME, object1.getName()), object1),
          createItemInfoForStorageObject(
              new StorageResourceId(BUCKET_NAME, object2.getName()), object2))
      .inOrder();
  assertThat(trackingHttpRequestInitializer.getAllRequestStrings())
      .containsExactly(
          listRequestString(
              BUCKET_NAME,
              /* includeTrailingDelimiter= */ true,
              prefix,
              /* maxResults= */ 1024,
              /* pageToken= */ null))
      .inOrder();
}
 
Example #17
Source File: GoogleCloudStorageImpl.java    From hadoop-connectors with Apache License 2.0 5 votes vote down vote up
/**
 * Helper for both listObjectNames and listObjectInfo that executes the actual API calls to get
 * paginated lists, accumulating the StorageObjects and String prefixes into the params {@code
 * listedObjects} and {@code listedPrefixes}.
 *
 * @param bucketName bucket name
 * @param objectNamePrefix object name prefix or null if all objects in the bucket are desired
 * @param delimiter delimiter to use (typically "/"), otherwise null
 * @param includeTrailingDelimiter whether to include prefix objects into the {@code
 *     listedObjects}
 * @param maxResults maximum number of results to return (total of both {@code listedObjects} and
 *     {@code listedPrefixes}), unlimited if negative or zero
 * @param listedObjects output parameter into which retrieved StorageObjects will be added
 * @param listedPrefixes output parameter into which retrieved prefixes will be added
 */
private void listStorageObjectsAndPrefixes(
    String bucketName,
    String objectNamePrefix,
    String delimiter,
    boolean includeTrailingDelimiter,
    long maxResults,
    List<StorageObject> listedObjects,
    List<String> listedPrefixes)
    throws IOException {
  logger.atFine().log(
      "listStorageObjectsAndPrefixes(%s, %s, %s, %s, %d)",
      bucketName, objectNamePrefix, delimiter, includeTrailingDelimiter, maxResults);

  checkArgument(
      listedObjects != null && listedObjects.isEmpty(),
      "Must provide a non-null empty container for listedObjects.");
  checkArgument(
      listedPrefixes != null && listedPrefixes.isEmpty(),
      "Must provide a non-null empty container for listedPrefixes.");

  Storage.Objects.List listObject =
      createListRequest(
          bucketName, objectNamePrefix, delimiter, includeTrailingDelimiter, maxResults);

  String pageToken = null;
  do {
    if (pageToken != null) {
      logger.atFine().log("listStorageObjectsAndPrefixes: next page %s", pageToken);
      listObject.setPageToken(pageToken);
    }
    pageToken =
        listStorageObjectsAndPrefixesPage(listObject, maxResults, listedObjects, listedPrefixes);
  } while (pageToken != null
      && getMaxRemainingResults(maxResults, listedPrefixes, listedObjects) > 0);
}
 
Example #18
Source File: GoogleCloudStorageImpl.java    From hadoop-connectors with Apache License 2.0 5 votes vote down vote up
/**
 * Helper for creating a Storage.Objects.Copy object ready for dispatch given a bucket and object
 * for an empty object to be created. Caller must already verify that {@code resourceId}
 * represents a StorageObject and not a bucket.
 */
private Storage.Objects.Insert prepareEmptyInsert(
    StorageResourceId resourceId, CreateObjectOptions createObjectOptions) throws IOException {
  Map<String, String> rewrittenMetadata = encodeMetadata(createObjectOptions.getMetadata());
  StorageObject object =
      new StorageObject()
          .setName(resourceId.getObjectName())
          .setMetadata(rewrittenMetadata)
          .setContentEncoding(createObjectOptions.getContentEncoding());

  // Ideally we'd use EmptyContent, but Storage requires an AbstractInputStreamContent and not
  // just an HttpContent, so we'll just use the next easiest thing.
  ByteArrayContent emptyContent =
      new ByteArrayContent(createObjectOptions.getContentType(), new byte[0]);
  Storage.Objects.Insert insertObject =
      configureRequest(
          gcs.objects().insert(resourceId.getBucketName(), object, emptyContent),
          resourceId.getBucketName());
  insertObject.setDisableGZipContent(true);
  clientRequestHelper.setDirectUploadEnabled(insertObject, true);

  if (resourceId.hasGenerationId()) {
    insertObject.setIfGenerationMatch(resourceId.getGenerationId());
  } else if (!createObjectOptions.overwriteExisting()) {
    insertObject.setIfGenerationMatch(0L);
  }
  return insertObject;
}
 
Example #19
Source File: GoogleCloudStorageTest.java    From hadoop-connectors with Apache License 2.0 5 votes vote down vote up
@Test
public void testListObjectInfoReturnPrefixes() throws IOException {
  String objectPrefix = "foo/bar/baz/";
  String delimiter = "/";
  String dir0Name = "foo/bar/baz/dir0/";
  String dir1Name = "foo/bar/baz/dir1/";
  StorageObject dir0 = newStorageObject(BUCKET_NAME, dir0Name);
  StorageObject dir1 = newStorageObject(BUCKET_NAME, dir1Name);

  List<StorageObject> objects =
      ImmutableList.of(newStorageObject(BUCKET_NAME, objectPrefix), dir0, dir1);

  MockHttpTransport transport =
      mockTransport(jsonDataResponse(new Objects().setItems(objects).setNextPageToken(null)));

  GoogleCloudStorage gcs = mockedGcs(transport);

  List<GoogleCloudStorageItemInfo> objectInfos =
      gcs.listObjectInfo(BUCKET_NAME, objectPrefix, delimiter);

  trackingHttpRequestInitializer.getAllRequestStrings();
  assertThat(objectInfos)
      .containsExactly(
          createItemInfoForStorageObject(new StorageResourceId(BUCKET_NAME, dir0Name), dir0),
          createItemInfoForStorageObject(new StorageResourceId(BUCKET_NAME, dir1Name), dir1))
      .inOrder();

  assertThat(trackingHttpRequestInitializer.getAllRequestStrings())
      .containsExactly(
          listRequestString(
              BUCKET_NAME,
              /* includeTrailingDelimiter= */ true,
              objectPrefix,
              /* maxResults= */ 1024,
              /* pageToken= */ null))
      .inOrder();
}
 
Example #20
Source File: GcsStorageService.java    From kayenta with Apache License 2.0 5 votes vote down vote up
private <T> T deserialize(Storage storage, StorageObject object, TypeReference<T> typeReference)
    throws IOException {
  ByteArrayOutputStream output = new java.io.ByteArrayOutputStream();
  Storage.Objects.Get getter = storage.objects().get(object.getBucket(), object.getName());
  getter.executeMediaAndDownloadTo(output);
  String json = output.toString("UTF8");

  return kayentaObjectMapper.readValue(json, typeReference);
}
 
Example #21
Source File: GcsUtilTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void testMakeRewriteOpsWithOptions() throws IOException {
  GcsOptions gcsOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = gcsOptions.getGcsUtil();
  gcsUtil.maxBytesRewrittenPerCall = 1337L;

  LinkedList<RewriteOp> rewrites =
      gcsUtil.makeRewriteOps(makeStrings("s", 1), makeStrings("d", 1));
  assertEquals(1, rewrites.size());

  RewriteOp rewrite = rewrites.pop();
  assertTrue(rewrite.getReadyToEnqueue());
  Storage.Objects.Rewrite request = rewrite.rewriteRequest;
  assertEquals(Long.valueOf(1337L), request.getMaxBytesRewrittenPerCall());
}
 
Example #22
Source File: GcsUtilTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void testRetryFileSizeNonBatch() throws IOException {
  GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = pipelineOptions.getGcsUtil();

  Storage mockStorage = Mockito.mock(Storage.class);
  gcsUtil.setStorageClient(mockStorage);

  Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class);
  Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class);

  BackOff mockBackOff =
      BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.withMaxRetries(2).backoff());

  when(mockStorage.objects()).thenReturn(mockStorageObjects);
  when(mockStorageObjects.get("testbucket", "testobject")).thenReturn(mockStorageGet);
  when(mockStorageGet.execute())
      .thenThrow(new SocketTimeoutException("SocketException"))
      .thenThrow(new SocketTimeoutException("SocketException"))
      .thenReturn(new StorageObject().setSize(BigInteger.valueOf(1000)));

  assertEquals(
      1000,
      gcsUtil
          .getObject(
              GcsPath.fromComponents("testbucket", "testobject"),
              mockBackOff,
              new FastNanoClockAndSleeper())
          .getSize()
          .longValue());
  assertEquals(BackOff.STOP, mockBackOff.nextBackOffMillis());
}
 
Example #23
Source File: GcsUtilTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Test
public void testAccessDeniedObjectThrowsIOException() throws IOException {
  GcsOptions pipelineOptions = gcsOptionsWithTestCredential();
  GcsUtil gcsUtil = pipelineOptions.getGcsUtil();

  Storage mockStorage = Mockito.mock(Storage.class);
  gcsUtil.setStorageClient(mockStorage);

  Storage.Objects mockStorageObjects = Mockito.mock(Storage.Objects.class);
  Storage.Objects.Get mockStorageGet = Mockito.mock(Storage.Objects.Get.class);

  GcsPath pattern = GcsPath.fromUri("gs://testbucket/testdirectory/accessdeniedfile");
  GoogleJsonResponseException expectedException =
      googleJsonResponseException(
          HttpStatusCodes.STATUS_CODE_FORBIDDEN,
          "Waves hand mysteriously",
          "These aren't the buckets you're looking for");

  when(mockStorage.objects()).thenReturn(mockStorageObjects);
  when(mockStorageObjects.get(pattern.getBucket(), pattern.getObject()))
      .thenReturn(mockStorageGet);
  when(mockStorageGet.execute()).thenThrow(expectedException);

  thrown.expect(IOException.class);
  thrown.expectMessage("Unable to get the file object for path");
  gcsUtil.expand(pattern);
}
 
Example #24
Source File: GcsFileSystem.java    From beam with Apache License 2.0 5 votes vote down vote up
/**
 * Expands a pattern into {@link MatchResult}.
 *
 * @throws IllegalArgumentException if {@code gcsPattern} does not contain globs.
 */
@VisibleForTesting
MatchResult expand(GcsPath gcsPattern) throws IOException {
  String prefix = GcsUtil.getNonWildcardPrefix(gcsPattern.getObject());
  Pattern p = Pattern.compile(GcsUtil.wildcardToRegexp(gcsPattern.getObject()));

  LOG.debug(
      "matching files in bucket {}, prefix {} against pattern {}",
      gcsPattern.getBucket(),
      prefix,
      p.toString());

  String pageToken = null;
  List<Metadata> results = new ArrayList<>();
  do {
    Objects objects = options.getGcsUtil().listObjects(gcsPattern.getBucket(), prefix, pageToken);
    if (objects.getItems() == null) {
      break;
    }

    // Filter objects based on the regex.
    for (StorageObject o : objects.getItems()) {
      String name = o.getName();
      // Skip directories, which end with a slash.
      if (p.matcher(name).matches() && !name.endsWith("/")) {
        LOG.debug("Matched object: {}", name);
        results.add(toMetadata(o));
      }
    }
    pageToken = objects.getNextPageToken();
  } while (pageToken != null);
  return MatchResult.create(Status.OK, results);
}
 
Example #25
Source File: LiveStateCheckerTest.java    From policyscanner with Apache License 2.0 5 votes vote down vote up
private void setUpGetFilesPage(List<String> objectNames) {
  // these are final classes, so use fakes instead of mocks.
  List<StorageObject> fakeItems = new ArrayList<>();
  for (String anObjectName : objectNames) {
    fakeItems.add(new StorageObject().setName(anObjectName));
  }

  Objects listObjects = new Objects().setItems(fakeItems);

  try {
    when(this.objectList.execute()).thenReturn(listObjects);
  } catch (IOException e) {
    fail("Failed to setup getFilesPage");
  }
}
 
Example #26
Source File: DesiredStateEnforcerTest.java    From policyscanner with Apache License 2.0 5 votes vote down vote up
private void setUpGetFilesPage(List<String> objectNames) {
  // these are final classes, so use fakes instead of mocks.
  List<StorageObject> fakeItems = new ArrayList<>();
  for (String anObjectName : objectNames) {
    fakeItems.add(new StorageObject().setName(anObjectName));
  }

  Objects listObjects = new Objects().setItems(fakeItems);

  try {
    when(this.objectList.execute()).thenReturn(listObjects);
  } catch (IOException e) {
    fail("Failed to setup getFilesPage");
  }
}
 
Example #27
Source File: OnDemandLiveStateCheckerTest.java    From policyscanner with Apache License 2.0 5 votes vote down vote up
private void setUpGetFilesPage(List<String> objectNames) {
  // these are final classes, so use fakes instead of mocks.
  List<StorageObject> fakeItems = new ArrayList<>();
  for (String anObjectName : objectNames) {
    fakeItems.add(new StorageObject().setName(anObjectName));
  }

  Objects listObjects = new Objects().setItems(fakeItems);

  try {
    when(this.objectList.execute()).thenReturn(listObjects);
  } catch (IOException e) {
    fail("Failed to setup getFilesPage");
  }
}
 
Example #28
Source File: GCSFilesSource.java    From policyscanner with Apache License 2.0 5 votes vote down vote up
/**
 * Obtain a list of files that reside in the repository directory.
 * This will yield all files, no matter how deeply nested they are in the directory.
 * @param fileNames The list which will be populated with the file names.
 * @param nextPageToken The token used to get the next page of the file list.
 * @return The token which will be used to get the next page of the file list.
 * @throws GeneralSecurityException Thrown if there's a permissions error using the GCS API.
 * @throws IOException Thrown if there's a IO error using the GCS API.
 */
String getFilesPage(List<String> fileNames, String nextPageToken)
    throws GeneralSecurityException, IOException, BucketAccessException {
  Objects objects =
      getStorageApiStub()
          .objects()
          .list(this.bucket)
          .setPageToken(nextPageToken)
          .setPrefix(this.repository + this.getDirDelimiter())
          .execute();
  if (objects == null || objects.getItems() == null) {
    String messageFormat = new StringBuilder()
        .append("Can't access objects in gs://%s/%s\n\n")
        .append("Check that your appengine-web.xml variables are setup correctly.\n")
        .append("In particular, check the correctness of the following variables:\n\n")
        .append("    POLICY_SCANNER_ORG_NAME\n")
        .append("    POLICY_SCANNER_INPUT_REPOSITORY_URL\n").toString();
    throw new BucketAccessException(String.format(messageFormat, this.bucket, this.repository));
  }
  for (int i = 0; i < objects.getItems().size(); ++i) {
    String fileName = objects.getItems().get(i).getName();
    if (!fileName.endsWith(getDirDelimiter())) {
      fileNames.add(fileName);
    }
  }
  return objects.getNextPageToken();
}
 
Example #29
Source File: GoogleCloudStorageImpl.java    From hadoop-connectors with Apache License 2.0 4 votes vote down vote up
/** See {@link GoogleCloudStorage#open(StorageResourceId)} for details about expected behavior. */
@Override
public SeekableByteChannel open(
    final StorageResourceId resourceId, GoogleCloudStorageReadOptions readOptions)
    throws IOException {
  logger.atFine().log("open(%s, %s)", resourceId, readOptions);
  Preconditions.checkArgument(
      resourceId.isStorageObject(), "Expected full StorageObject id, got %s", resourceId);

  if (storageOptions.isGrpcEnabled()) {
    return GoogleCloudStorageGrpcReadChannel.open(
        storageStubProvider.getBlockingStub(),
        resourceId.getBucketName(),
        resourceId.getObjectName(),
        readOptions);
  }

  // The underlying channel doesn't initially read data, which means that we won't see a
  // FileNotFoundException until read is called. As a result, in order to find out if the object
  // exists, we'll need to do an RPC (metadata or data). A metadata check should be a less
  // expensive operation than a read data operation.
  GoogleCloudStorageItemInfo info;
  if (readOptions.getFastFailOnNotFound()) {
    info = getItemInfo(resourceId);
    if (!info.exists()) {
      throw createFileNotFoundException(
          resourceId.getBucketName(), resourceId.getObjectName(), /* cause= */ null);
    }
  } else {
    info = null;
  }

  return new GoogleCloudStorageReadChannel(
      gcs, resourceId, errorExtractor, clientRequestHelper, readOptions) {

    @Override
    @Nullable
    protected GoogleCloudStorageItemInfo getInitialMetadata() {
      return info;
    }

    @Override
    protected Storage.Objects.Get createRequest() throws IOException {
      return configureRequest(super.createRequest(), resourceId.getBucketName());
    }
  };
}
 
Example #30
Source File: GcsFileSystemTest.java    From beam with Apache License 2.0 4 votes vote down vote up
@Test
public void testMatch() throws Exception {
  Objects modelObjects = new Objects();
  List<StorageObject> items = new ArrayList<>();
  // A directory
  items.add(new StorageObject().setBucket("testbucket").setName("testdirectory/"));

  // Files within the directory
  items.add(createStorageObject("gs://testbucket/testdirectory/file1name", 1L /* fileSize */));
  items.add(createStorageObject("gs://testbucket/testdirectory/file2name", 2L /* fileSize */));
  items.add(createStorageObject("gs://testbucket/testdirectory/file3name", 3L /* fileSize */));
  items.add(createStorageObject("gs://testbucket/testdirectory/file4name", 4L /* fileSize */));
  items.add(createStorageObject("gs://testbucket/testdirectory/otherfile", 5L /* fileSize */));
  items.add(createStorageObject("gs://testbucket/testdirectory/anotherfile", 6L /* fileSize */));

  modelObjects.setItems(items);
  when(mockGcsUtil.listObjects(eq("testbucket"), anyString(), isNull(String.class)))
      .thenReturn(modelObjects);

  List<GcsPath> gcsPaths =
      ImmutableList.of(
          GcsPath.fromUri("gs://testbucket/testdirectory/non-exist-file"),
          GcsPath.fromUri("gs://testbucket/testdirectory/otherfile"));

  when(mockGcsUtil.getObjects(eq(gcsPaths)))
      .thenReturn(
          ImmutableList.of(
              StorageObjectOrIOException.create(new FileNotFoundException()),
              StorageObjectOrIOException.create(
                  createStorageObject("gs://testbucket/testdirectory/otherfile", 4L))));

  List<String> specs =
      ImmutableList.of(
          "gs://testbucket/testdirectory/file[1-3]*",
          "gs://testbucket/testdirectory/non-exist-file",
          "gs://testbucket/testdirectory/otherfile");
  List<MatchResult> matchResults = gcsFileSystem.match(specs);
  assertEquals(3, matchResults.size());
  assertEquals(Status.OK, matchResults.get(0).status());
  assertThat(
      ImmutableList.of(
          "gs://testbucket/testdirectory/file1name",
          "gs://testbucket/testdirectory/file2name",
          "gs://testbucket/testdirectory/file3name"),
      contains(toFilenames(matchResults.get(0)).toArray()));
  assertEquals(Status.NOT_FOUND, matchResults.get(1).status());
  assertEquals(Status.OK, matchResults.get(2).status());
  assertThat(
      ImmutableList.of("gs://testbucket/testdirectory/otherfile"),
      contains(toFilenames(matchResults.get(2)).toArray()));
}