com.google.appengine.tools.cloudstorage.GcsFilename Java Examples

The following examples show how to use com.google.appengine.tools.cloudstorage.GcsFilename. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LocalRawGcsService.java    From appengine-gcs-client with Apache License 2.0 6 votes vote down vote up
@Override
public GcsFileMetadata getObjectMetadata(GcsFilename filename, long timeoutMillis)
    throws IOException {
  ensureInitialized();
  Entity entity;
  try {
    entity = datastore.get(null, makeKey(filename));
    return createGcsFileMetadata(entity, filename);
  } catch (EntityNotFoundException ex1) {
    try {
      entity = datastore.get(null, makeBlobstoreKey(filename));
      return createGcsFileMetadataFromBlobstore(entity, filename);
    } catch (EntityNotFoundException ex2) {
      return null;
    }
  }
}
 
Example #2
Source File: BrdaCopyAction.java    From nomulus with Apache License 2.0 6 votes vote down vote up
private void copyAsRyde() throws IOException, PGPException {
  String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, THIN, 1, 0);
  GcsFilename xmlFilename = new GcsFilename(stagingBucket, prefix + ".xml.ghostryde");
  GcsFilename xmlLengthFilename = new GcsFilename(stagingBucket, prefix + ".xml.length");
  GcsFilename rydeFile = new GcsFilename(brdaBucket, prefix + ".ryde");
  GcsFilename sigFile = new GcsFilename(brdaBucket, prefix + ".sig");

  long xmlLength = readXmlLength(xmlLengthFilename);

  logger.atInfo().log("Writing %s and %s", rydeFile, sigFile);
  try (InputStream gcsInput = gcsUtils.openInputStream(xmlFilename);
      InputStream ghostrydeDecoder = Ghostryde.decoder(gcsInput, stagingDecryptionKey);
      OutputStream rydeOut = gcsUtils.openOutputStream(rydeFile);
      OutputStream sigOut = gcsUtils.openOutputStream(sigFile);
      RydeEncoder rydeEncoder = new RydeEncoder.Builder()
          .setRydeOutput(rydeOut, receiverKey)
          .setSignatureOutput(sigOut, signingKey)
          .setFileMetadata(prefix, xmlLength, watermark)
          .build()) {
    ByteStreams.copy(ghostrydeDecoder, rydeEncoder);
  }
}
 
Example #3
Source File: ObjectifyStorageIo.java    From appinventor-extensions with Apache License 2.0 6 votes vote down vote up
private FileData createRawFile(Key<ProjectData> projectKey, FileData.RoleEnum role,
  String userId, String fileName, byte[] content) throws ObjectifyException, IOException {
  validateGCS();
  FileData file = new FileData();
  file.fileName = fileName;
  file.projectKey = projectKey;
  file.role = role;
  file.userId = userId;
  if (useGCSforFile(fileName, content.length)) {
    file.isGCS = true;
    file.gcsName = makeGCSfileName(fileName, projectKey.getId());
    GcsOutputChannel outputChannel =
      gcsService.createOrReplace(new GcsFilename(GCS_BUCKET_NAME, file.gcsName), GcsFileOptions.getDefaultInstance());
    outputChannel.write(ByteBuffer.wrap(content));
    outputChannel.close();
  } else {
    file.content = content;
  }
  return file;
}
 
Example #4
Source File: ObjectifyStorageIo.java    From appinventor-extensions with Apache License 2.0 6 votes vote down vote up
@Override
public InputStream openTempFile(String fileName) throws IOException {
  if (!fileName.startsWith("__TEMP__")) {
    throw new RuntimeException("deleteTempFile (" + fileName + ") Invalid File Name");
  }
  GcsFilename gcsFileName = new GcsFilename(GCS_BUCKET_NAME, fileName);
  int fileSize = (int) gcsService.getMetadata(gcsFileName).getLength();
  ByteBuffer resultBuffer = ByteBuffer.allocate(fileSize);
  GcsInputChannel readChannel = gcsService.openReadChannel(gcsFileName, 0);
  int bytesRead = 0;
  try {
    while (bytesRead < fileSize) {
      bytesRead += readChannel.read(resultBuffer);
    }
  } finally {
    readChannel.close();
  }
  return new ByteArrayInputStream(resultBuffer.array());
}
 
Example #5
Source File: Spec11RegistrarThreatMatchesParser.java    From nomulus with Apache License 2.0 6 votes vote down vote up
public Optional<LocalDate> getPreviousDateWithMatches(LocalDate date) {
  LocalDate yesterday = date.minusDays(1);
  GcsFilename gcsFilename = getGcsFilename(yesterday);
  if (gcsUtils.existsAndNotEmpty(gcsFilename)) {
    return Optional.of(yesterday);
  }
  logger.atWarning().log("Could not find previous file from date %s", yesterday);

  for (LocalDate dateToCheck = yesterday.minusDays(1);
      !dateToCheck.isBefore(date.minusMonths(1));
      dateToCheck = dateToCheck.minusDays(1)) {
    gcsFilename = getGcsFilename(dateToCheck);
    if (gcsUtils.existsAndNotEmpty(gcsFilename)) {
      return Optional.of(dateToCheck);
    }
  }
  return Optional.empty();
}
 
Example #6
Source File: IcannReportingStager.java    From nomulus with Apache License 2.0 6 votes vote down vote up
/** Stores a report on GCS, returning the name of the file stored. */
private String saveReportToGcs(
    String tld, YearMonth yearMonth, String subdir, String reportCsv, ReportType reportType)
    throws IOException {
  // Upload resulting CSV file to GCS
  byte[] reportBytes = reportCsv.getBytes(UTF_8);
  String reportFilename =
      String.format(
          "%s-%s-%s.csv",
          tld,
          Ascii.toLowerCase(reportType.toString()),
          DateTimeFormat.forPattern("yyyyMM").print(yearMonth));
  String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
  final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
  gcsUtils.createFromBytes(gcsFilename, reportBytes);
  logger.atInfo().log("Wrote %d bytes to file location %s", reportBytes.length, gcsFilename);
  return reportFilename;
}
 
Example #7
Source File: OauthRawGcsService.java    From appengine-gcs-client with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
static URL makeUrl(GcsFilename filename, @Nullable Map<String, String> queryStrings) {
  String path = makePath(filename);
  try {
    StringBuilder url =
        new StringBuilder().append(new URI("https", STORAGE_API_HOSTNAME, path, null));
    if (queryStrings != null && !queryStrings.isEmpty()) {
      url.append('?');
      for (Map.Entry<String, String> entry : queryStrings.entrySet()) {
        url.append(URLEncoder.encode(entry.getKey(), UTF_8.name()));
        if (entry.getValue() != null) {
          url.append('=').append(URLEncoder.encode(entry.getValue(), UTF_8.name()));
        }
        url.append('&');
      }
      url.setLength(url.length() - 1);
    }
    return new URL(url.toString());
  } catch (MalformedURLException | URISyntaxException | UnsupportedEncodingException e) {
    throw new RuntimeException(
        "Could not create a URL for " + filename + " with query " + queryStrings, e);
  }
}
 
Example #8
Source File: OauthRawGcsService.java    From appengine-gcs-client with Apache License 2.0 6 votes vote down vote up
@Override
public RawGcsCreationToken beginObjectCreation(
    GcsFilename filename, GcsFileOptions options, long timeoutMillis) throws IOException {
  HTTPRequest req = makeRequest(filename, null, POST, timeoutMillis);
  req.setHeader(RESUMABLE_HEADER);
  addOptionsHeaders(req, options);
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  if (resp.getResponseCode() == 201) {
    String location = URLFetchUtils.getSingleHeader(resp, LOCATION);
    String queryString = new URL(location).getQuery();
    Preconditions.checkState(
        queryString != null, LOCATION + " header," + location + ", witout a query string");
    Map<String, String> params = Splitter.on('&').withKeyValueSeparator('=').split(queryString);
    Preconditions.checkState(params.containsKey(UPLOAD_ID),
        LOCATION + " header," + location + ", has a query string without " + UPLOAD_ID);
    return new GcsRestCreationToken(filename, params.get(UPLOAD_ID), 0);
  } else {
    throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
}
 
Example #9
Source File: OauthRawGcsService.java    From appengine-gcs-client with Apache License 2.0 6 votes vote down vote up
@Override
public void copyObject(GcsFilename source, GcsFilename dest, GcsFileOptions fileOptions,
    long timeoutMillis) throws IOException {
  HTTPRequest req = makeRequest(dest, null, PUT, timeoutMillis);
  req.setHeader(new HTTPHeader(X_GOOG_COPY_SOURCE, makePath(source)));
  if (fileOptions != null) {
    req.setHeader(REPLACE_METADATA_HEADER);
    addOptionsHeaders(req, fileOptions);
  }
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  if (resp.getResponseCode() != 200) {
    throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
}
 
Example #10
Source File: RdeReportAction.java    From nomulus with Apache License 2.0 6 votes vote down vote up
@Override
public void runWithLock(DateTime watermark) throws Exception {
  Cursor cursor =
      ofy().load().key(Cursor.createKey(CursorType.RDE_UPLOAD, Registry.get(tld))).now();
  loadAndCompare(cursor, tld);
  DateTime cursorTime = getCursorTimeOrStartOfTime(cursor);
  if (isBeforeOrAt(cursorTime, watermark)) {
    throw new NoContentException(
        String.format(
            "Waiting on RdeUploadAction for TLD %s to send %s report; "
                + "last upload completion was at %s",
            tld, watermark, cursorTime));
  }
  String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, 0);
  GcsFilename reportFilename = new GcsFilename(bucket, prefix + "-report.xml.ghostryde");
  verify(gcsUtils.existsAndNotEmpty(reportFilename), "Missing file: %s", reportFilename);
  reporter.send(readReportFromGcs(reportFilename));
  response.setContentType(PLAIN_TEXT_UTF_8);
  response.setPayload(String.format("OK %s %s\n", tld, watermark));
}
 
Example #11
Source File: OauthRawGcsServiceTest.java    From appengine-gcs-client with Apache License 2.0 6 votes vote down vote up
@Test
public void makeUrlShouldCorrectlyEncodeQueryString() {
  Map<String, String> queryStrings = new LinkedHashMap<>();
  queryStrings.put("upload_id", "} 20");
  queryStrings.put("composed", null);
  queryStrings.put("val=ue", "%7D&");
  queryStrings.put("regular", "val");
  queryStrings.put("k e+y", "=v a+lu&e=");
  String url = makeUrl(new GcsFilename(BUCKET, "object"), queryStrings).toString();
  String expected = URL_PREFIX + "object?"
      + "upload_id=%7D+20&"
      + "composed&"
      + "val%3Due=%257D%26&"
      + "regular=val&"
      + "k+e%2By=%3Dv+a%2Blu%26e%3D";
  assertEquals(expected, url);
}
 
Example #12
Source File: OauthRawGcsService.java    From appengine-gcs-client with Apache License 2.0 6 votes vote down vote up
@Override
public void composeObject(Iterable<String> source, GcsFilename dest, long timeoutMillis)
    throws IOException {
  StringBuilder xmlContent = new StringBuilder(Iterables.size(source) * 50);
  Escaper escaper = XmlEscapers.xmlContentEscaper();
  xmlContent.append("<ComposeRequest>");
  for (String srcFileName : source) {
    xmlContent.append("<Component><Name>")
        .append(escaper.escape(srcFileName))
        .append("</Name></Component>");
  }
  xmlContent.append("</ComposeRequest>");
  HTTPRequest req = makeRequest(
      dest, COMPOSE_QUERY_STRINGS, PUT, timeoutMillis, xmlContent.toString().getBytes(UTF_8));
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  if (resp.getResponseCode() != 200) {
    throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
}
 
Example #13
Source File: LocalRawGcsService.java    From appengine-gcs-client with Apache License 2.0 6 votes vote down vote up
@Override
public void composeObject(Iterable<String> source, GcsFilename dest, long timeoutMillis)
    throws IOException {
  ensureInitialized();
  int size = Iterables.size(source);
  if (size > 32) {
    throw new IOException("Compose attempted with too many components. Limit is 32");
  }
  if (size < 2) {
    throw new IOException("You must provide at least two source components.");
  }
  Token token = beginObjectCreation(dest, GcsFileOptions.getDefaultInstance(), timeoutMillis);

  for (String filename : source) {
    GcsFilename sourceGcsFilename = new GcsFilename(dest.getBucketName(), filename);
    appendFileContentsToToken(sourceGcsFilename, token);
  }
  finishObjectCreation(token, ByteBuffer.allocate(0), timeoutMillis);
}
 
Example #14
Source File: GalleryServiceImpl.java    From appinventor-extensions with Apache License 2.0 6 votes vote down vote up
/**
 * delete aia file based on given gallery id
 * @param galleryId gallery id
 */
private void deleteAIA(long galleryId) {
  try {
    //String galleryKey = GalleryApp.getSourceKey(galleryId);
    GallerySettings settings = loadGallerySettings();
    String galleryKey = settings.getSourceKey(galleryId);

    // setup cloud
    GcsService gcsService = GcsServiceFactory.createGcsService();
    //GcsFilename filename = new GcsFilename(GalleryApp.GALLERYBUCKET, galleryKey);
    GcsFilename filename = new GcsFilename(settings.getBucket(), galleryKey);
    gcsService.delete(filename);
  } catch (IOException e) {
    // TODO Auto-generated catch block
    LOG.log(Level.INFO, "FAILED GCS delete");
    e.printStackTrace();
  }
}
 
Example #15
Source File: ExportDomainListsActionTest.java    From nomulus with Apache License 2.0 6 votes vote down vote up
@Test
public void test_outputsOnlyDomainsOnRealTlds() throws Exception {
  persistActiveDomain("onetwo.tld");
  persistActiveDomain("rudnitzky.tld");
  persistActiveDomain("wontgo.testtld");
  runMapreduce();
  GcsFilename existingFile = new GcsFilename("outputbucket", "tld.txt");
  String tlds = new String(readGcsFile(gcsService, existingFile), UTF_8).trim();
  // Check that it only contains the domains on the real TLD, and not the test one.
  assertThat(tlds).isEqualTo("onetwo.tld\nrudnitzky.tld");
  // Make sure that the test TLD file wasn't written out.
  GcsFilename nonexistentFile = new GcsFilename("outputbucket", "testtld.txt");
  assertThrows(FileNotFoundException.class, () -> readGcsFile(gcsService, nonexistentFile));
  ListResult ls = gcsService.list("outputbucket", ListOptions.DEFAULT);
  assertThat(ls.next().getName()).isEqualTo("tld.txt");
  // Make sure that no other files were written out.
  assertThat(ls.hasNext()).isFalse();
  verifyExportedToDrive("brouhaha", "onetwo.tld\nrudnitzky.tld");
  verifyNoMoreInteractions(driveConnection);
}
 
Example #16
Source File: RestoreCommitLogsActionTest.java    From nomulus with Apache License 2.0 6 votes vote down vote up
private Iterable<ImmutableObject> saveDiffFile(
    CommitLogCheckpoint checkpoint, ImmutableObject... entities) throws IOException {
  DateTime now = checkpoint.getCheckpointTime();
  List<ImmutableObject> allEntities = Lists.asList(checkpoint, entities);
  ByteArrayOutputStream output = new ByteArrayOutputStream();
  for (ImmutableObject entity : allEntities) {
    serializeEntity(entity, output);
  }
  gcsService.createOrReplace(
      new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + now),
      new GcsFileOptions.Builder()
          .addUserMetadata(LOWER_BOUND_CHECKPOINT, now.minusMinutes(1).toString())
          .build(),
      ByteBuffer.wrap(output.toByteArray()));
  return allEntities;
}
 
Example #17
Source File: OauthRawGcsService.java    From appengine-gcs-client with Apache License 2.0 6 votes vote down vote up
@Override
public GcsFileMetadata getObjectMetadata(GcsFilename filename, long timeoutMillis)
    throws IOException {
  HTTPRequest req = makeRequest(filename, null, HEAD, timeoutMillis);
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  int responseCode = resp.getResponseCode();
  if (responseCode == 404) {
    return null;
  }
  if (responseCode != 200) {
    throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
  return getMetadataFromResponse(
      filename, resp, getLengthFromHeader(resp, X_GOOG_CONTENT_LENGTH));
}
 
Example #18
Source File: RdeUploadActionTest.java    From nomulus with Apache License 2.0 6 votes vote down vote up
@Test
public void testRunWithLock_copiesOnGcs() throws Exception {
  int port = sftpd.serve("user", "password", folder.getRoot());
  URI uploadUrl = URI.create(String.format("sftp://user:password@localhost:%d/", port));
  DateTime stagingCursor = DateTime.parse("2010-10-18TZ");
  DateTime uploadCursor = DateTime.parse("2010-10-17TZ");
  persistResource(Cursor.create(RDE_STAGING, stagingCursor, Registry.get("tld")));
  createAction(uploadUrl).runWithLock(uploadCursor);
  assertThat(response.getStatus()).isEqualTo(200);
  assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
  assertThat(response.getPayload()).isEqualTo("OK tld 2010-10-17T00:00:00.000Z\n");
  assertNoTasksEnqueued("rde-upload");
  // Assert that both files are written to SFTP and GCS, and that the contents are identical.
  String rydeFilename = "tld_2010-10-17_full_S1_R0.ryde";
  String sigFilename = "tld_2010-10-17_full_S1_R0.sig";
  assertThat(folder.getRoot().list()).asList().containsExactly(rydeFilename, sigFilename);
  assertThat(readGcsFile(gcsService, new GcsFilename("bucket", rydeFilename)))
      .isEqualTo(Files.toByteArray(new File(folder.getRoot(), rydeFilename)));
  assertThat(readGcsFile(gcsService, new GcsFilename("bucket", sigFilename)))
      .isEqualTo(Files.toByteArray(new File(folder.getRoot(), sigFilename)));
}
 
Example #19
Source File: OauthRawGcsService.java    From appengine-gcs-client with Apache License 2.0 6 votes vote down vote up
/** True if deleted, false if not found. */
@Override
public boolean deleteObject(GcsFilename filename, long timeoutMillis) throws IOException {
  HTTPRequest req = makeRequest(filename, null, DELETE, timeoutMillis);
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  switch (resp.getResponseCode()) {
    case 204:
      return true;
    case 404:
      return false;
    default:
      throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
}
 
Example #20
Source File: RdeStagingActionTest.java    From nomulus with Apache License 2.0 6 votes vote down vote up
@Test
public void testMapReduce_idnTables_goInDeposit() throws Exception {
  clock.setTo(DateTime.parse("1999-12-31TZ"));
  createTldWithEscrowEnabled("fop");
  makeDomainBase(clock, "fop");

  clock.setTo(DateTime.parse("2000-01-01TZ"));
  action.run();
  executeTasksUntilEmpty("mapreduce", clock);

  GcsFilename filename = new GcsFilename("rde-bucket", "fop_2000-01-01_full_S1_R0.xml.ghostryde");
  XjcRdeDeposit deposit =
      unmarshal(
          XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, filename), decryptKey));
  XjcRdeDomain domain = extractAndRemoveContentWithType(XjcRdeDomain.class, deposit);
  XjcRdeIdn firstIdn = extractAndRemoveContentWithType(XjcRdeIdn.class, deposit);
  XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit);

  assertThat(domain.getIdnTableId()).isEqualTo("extended_latin");
  assertThat(firstIdn.getId()).isEqualTo("extended_latin");
  assertThat(firstIdn.getUrl()).isEqualTo(EXTENDED_LATIN.getTable().getUrl().toString());
  assertThat(firstIdn.getUrlPolicy()).isEqualTo(EXTENDED_LATIN.getTable().getPolicy().toString());
  assertThat(mapifyCounts(header))
      .containsEntry(RdeResourceType.IDN.getUri(), (long) IdnTableEnum.values().length);
}
 
Example #21
Source File: CopyDetailReportsActionTest.java    From nomulus with Apache License 2.0 6 votes vote down vote up
@Test
public void testSuccess_transientIOException_retries() throws IOException {
  writeGcsFile(
      gcsService,
      new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
      "hola,mundo\n3,4".getBytes(UTF_8));
  when(driveConnection.createOrUpdateFile(any(), any(), any(), any()))
      .thenThrow(new IOException("expected"))
      .thenReturn("success");

  action.run();
  verify(driveConnection, times(2))
      .createOrUpdateFile(
          "invoice_details_2017-10_TheRegistrar_hello.csv",
          MediaType.CSV_UTF_8,
          "0B-12345",
          "hola,mundo\n3,4".getBytes(UTF_8));
  assertThat(response.getStatus()).isEqualTo(SC_OK);
  assertThat(response.getContentType()).isEqualTo(MediaType.PLAIN_TEXT_UTF_8);
  assertThat(response.getPayload()).isEqualTo("Copied detail reports.\n");
}
 
Example #22
Source File: CopyDetailReportsActionTest.java    From nomulus with Apache License 2.0 6 votes vote down vote up
@Test
public void testSuccess_nonDetailReportFiles_notSent() throws IOException{
  writeGcsFile(
      gcsService,
      new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
      "hola,mundo\n3,4".getBytes(UTF_8));

  writeGcsFile(
      gcsService,
      new GcsFilename("test-bucket", "results/not_a_detail_report_2017-10_TheRegistrar_test.csv"),
      "hello,world\n1,2".getBytes(UTF_8));
  action.run();
  verify(driveConnection)
      .createOrUpdateFile(
          "invoice_details_2017-10_TheRegistrar_hello.csv",
          MediaType.CSV_UTF_8,
          "0B-12345",
          "hola,mundo\n3,4".getBytes(UTF_8));
  // Verify we didn't copy the non-detail report file.
  verifyNoMoreInteractions(driveConnection);
  assertThat(response.getStatus()).isEqualTo(SC_OK);
  assertThat(response.getContentType()).isEqualTo(MediaType.PLAIN_TEXT_UTF_8);
  assertThat(response.getPayload()).isEqualTo("Copied detail reports.\n");
}
 
Example #23
Source File: LocalRawGcsService.java    From appengine-gcs-client with Apache License 2.0 6 votes vote down vote up
@Override
public void copyObject(GcsFilename source, GcsFilename dest, GcsFileOptions fileOptions,
    long timeoutMillis) throws IOException {
  ensureInitialized();
  GcsFileMetadata meta = getObjectMetadata(source, timeoutMillis);
  if (meta == null) {
    throw new FileNotFoundException(this + ": No such file: " + source);
  }
  if (fileOptions == null) {
    fileOptions = meta.getOptions();
  }

  Token token = beginObjectCreation(dest, fileOptions, timeoutMillis);
  appendFileContentsToToken(source, token);
  finishObjectCreation(token, ByteBuffer.allocate(0), timeoutMillis);
}
 
Example #24
Source File: IcannReportingUploadActionTest.java    From nomulus with Apache License 2.0 6 votes vote down vote up
@Test
public void testSuccess_advancesCursor() throws Exception {
  writeGcsFile(
      gcsService,
      new GcsFilename("basin/icann/monthly/2006-06", "tld-activity-200606.csv"),
      PAYLOAD_SUCCESS);
  when(mockReporter.send(PAYLOAD_SUCCESS, "tld-activity-200606.csv")).thenReturn(true);
  IcannReportingUploadAction action = createAction();
  action.run();
  ofy().clearSessionCache();
  Cursor cursor =
      ofy()
          .load()
          .key(Cursor.createKey(CursorType.ICANN_UPLOAD_ACTIVITY, Registry.get("tld")))
          .now();
  assertThat(cursor.getCursorTime()).isEqualTo(DateTime.parse("2006-08-01TZ"));
}
 
Example #25
Source File: CopyDetailReportsActionTest.java    From nomulus with Apache License 2.0 5 votes vote down vote up
@Test
public void testFail_noRegistrarFolderId_doesntCopy() throws IOException {
  persistResource(loadRegistrar("TheRegistrar").asBuilder().setDriveFolderId(null).build());
  writeGcsFile(
      gcsService,
      new GcsFilename(
          "test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
      "hola,mundo\n3,4".getBytes(UTF_8));
  action.run();
  verifyZeroInteractions(driveConnection);
}
 
Example #26
Source File: GcsDiffFileListerTest.java    From nomulus with Apache License 2.0 5 votes vote down vote up
private void addGcsFile(int fileAge, int prevAge) throws IOException {
  gcsService.createOrReplace(
      new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + now.minusMinutes(fileAge)),
      new GcsFileOptions.Builder()
          .addUserMetadata(LOWER_BOUND_CHECKPOINT, now.minusMinutes(prevAge).toString())
          .build(),
      ByteBuffer.wrap(new byte[]{1, 2, 3}));
}
 
Example #27
Source File: GcsExampleServlet.java    From appengine-gcs-client with Apache License 2.0 5 votes vote down vote up
private GcsFilename getFileName(HttpServletRequest req) {
  String[] splits = req.getRequestURI().split("/", 4);
  if (!splits[0].equals("") || !splits[1].equals("gcs")) {
    throw new IllegalArgumentException("The URL is not formed as expected. " +
        "Expecting /gcs/<bucket>/<object>");
  }
  return new GcsFilename(splits[2], splits[3]);
}
 
Example #28
Source File: LocalRawGcsService.java    From appengine-gcs-client with Apache License 2.0 5 votes vote down vote up
@Override
public Token beginObjectCreation(GcsFilename filename, GcsFileOptions options, long timeoutMillis)
    throws IOException {
  ensureInitialized();
  inMemoryData.put(filename, new ArrayList<ByteBuffer>());
  return new Token(filename, options, 0);
}
 
Example #29
Source File: IcannReportingUploadAction.java    From nomulus with Apache License 2.0 5 votes vote down vote up
private void verifyFileExists(GcsFilename gcsFilename) {
  checkArgument(
      gcsUtils.existsAndNotEmpty(gcsFilename),
      "Object %s in bucket %s not found",
      gcsFilename.getObjectName(),
      gcsFilename.getBucketName());
}
 
Example #30
Source File: ObjectifyStorageIo.java    From appinventor-extensions with Apache License 2.0 5 votes vote down vote up
@Override
public void deleteTempFile(String fileName) throws IOException {
  if (!fileName.startsWith("__TEMP__")) {
    throw new RuntimeException("deleteTempFile (" + fileName + ") Invalid File Name");
  }
  gcsService.delete(new GcsFilename(GCS_BUCKET_NAME, fileName));
}