Java Code Examples for com.google.appengine.tools.cloudstorage.GcsFilename

The following examples show how to use com.google.appengine.tools.cloudstorage.GcsFilename. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: appengine-gcs-client   Source File: OauthRawGcsService.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
static URL makeUrl(GcsFilename filename, @Nullable Map<String, String> queryStrings) {
  String path = makePath(filename);
  try {
    StringBuilder url =
        new StringBuilder().append(new URI("https", STORAGE_API_HOSTNAME, path, null));
    if (queryStrings != null && !queryStrings.isEmpty()) {
      url.append('?');
      for (Map.Entry<String, String> entry : queryStrings.entrySet()) {
        url.append(URLEncoder.encode(entry.getKey(), UTF_8.name()));
        if (entry.getValue() != null) {
          url.append('=').append(URLEncoder.encode(entry.getValue(), UTF_8.name()));
        }
        url.append('&');
      }
      url.setLength(url.length() - 1);
    }
    return new URL(url.toString());
  } catch (MalformedURLException | URISyntaxException | UnsupportedEncodingException e) {
    throw new RuntimeException(
        "Could not create a URL for " + filename + " with query " + queryStrings, e);
  }
}
 
Example 2
Source Project: appinventor-extensions   Source File: GalleryServiceImpl.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * delete aia file based on given gallery id
 * @param galleryId gallery id
 */
private void deleteAIA(long galleryId) {
  try {
    //String galleryKey = GalleryApp.getSourceKey(galleryId);
    GallerySettings settings = loadGallerySettings();
    String galleryKey = settings.getSourceKey(galleryId);

    // setup cloud
    GcsService gcsService = GcsServiceFactory.createGcsService();
    //GcsFilename filename = new GcsFilename(GalleryApp.GALLERYBUCKET, galleryKey);
    GcsFilename filename = new GcsFilename(settings.getBucket(), galleryKey);
    gcsService.delete(filename);
  } catch (IOException e) {
    // TODO Auto-generated catch block
    LOG.log(Level.INFO, "FAILED GCS delete");
    e.printStackTrace();
  }
}
 
Example 3
Source Project: appinventor-extensions   Source File: ObjectifyStorageIo.java    License: Apache License 2.0 6 votes vote down vote up
private FileData createRawFile(Key<ProjectData> projectKey, FileData.RoleEnum role,
  String userId, String fileName, byte[] content) throws ObjectifyException, IOException {
  validateGCS();
  FileData file = new FileData();
  file.fileName = fileName;
  file.projectKey = projectKey;
  file.role = role;
  file.userId = userId;
  if (useGCSforFile(fileName, content.length)) {
    file.isGCS = true;
    file.gcsName = makeGCSfileName(fileName, projectKey.getId());
    GcsOutputChannel outputChannel =
      gcsService.createOrReplace(new GcsFilename(GCS_BUCKET_NAME, file.gcsName), GcsFileOptions.getDefaultInstance());
    outputChannel.write(ByteBuffer.wrap(content));
    outputChannel.close();
  } else {
    file.content = content;
  }
  return file;
}
 
Example 4
Source Project: appinventor-extensions   Source File: ObjectifyStorageIo.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public InputStream openTempFile(String fileName) throws IOException {
  if (!fileName.startsWith("__TEMP__")) {
    throw new RuntimeException("deleteTempFile (" + fileName + ") Invalid File Name");
  }
  GcsFilename gcsFileName = new GcsFilename(GCS_BUCKET_NAME, fileName);
  int fileSize = (int) gcsService.getMetadata(gcsFileName).getLength();
  ByteBuffer resultBuffer = ByteBuffer.allocate(fileSize);
  GcsInputChannel readChannel = gcsService.openReadChannel(gcsFileName, 0);
  int bytesRead = 0;
  try {
    while (bytesRead < fileSize) {
      bytesRead += readChannel.read(resultBuffer);
    }
  } finally {
    readChannel.close();
  }
  return new ByteArrayInputStream(resultBuffer.array());
}
 
Example 5
Source Project: nomulus   Source File: IcannReportingStager.java    License: Apache License 2.0 6 votes vote down vote up
/** Stores a report on GCS, returning the name of the file stored. */
private String saveReportToGcs(
    String tld, YearMonth yearMonth, String subdir, String reportCsv, ReportType reportType)
    throws IOException {
  // Upload resulting CSV file to GCS
  byte[] reportBytes = reportCsv.getBytes(UTF_8);
  String reportFilename =
      String.format(
          "%s-%s-%s.csv",
          tld,
          Ascii.toLowerCase(reportType.toString()),
          DateTimeFormat.forPattern("yyyyMM").print(yearMonth));
  String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
  final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
  gcsUtils.createFromBytes(gcsFilename, reportBytes);
  logger.atInfo().log("Wrote %d bytes to file location %s", reportBytes.length, gcsFilename);
  return reportFilename;
}
 
Example 6
public Optional<LocalDate> getPreviousDateWithMatches(LocalDate date) {
  LocalDate yesterday = date.minusDays(1);
  GcsFilename gcsFilename = getGcsFilename(yesterday);
  if (gcsUtils.existsAndNotEmpty(gcsFilename)) {
    return Optional.of(yesterday);
  }
  logger.atWarning().log("Could not find previous file from date %s", yesterday);

  for (LocalDate dateToCheck = yesterday.minusDays(1);
      !dateToCheck.isBefore(date.minusMonths(1));
      dateToCheck = dateToCheck.minusDays(1)) {
    gcsFilename = getGcsFilename(dateToCheck);
    if (gcsUtils.existsAndNotEmpty(gcsFilename)) {
      return Optional.of(dateToCheck);
    }
  }
  return Optional.empty();
}
 
Example 7
Source Project: nomulus   Source File: BrdaCopyAction.java    License: Apache License 2.0 6 votes vote down vote up
private void copyAsRyde() throws IOException, PGPException {
  String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, THIN, 1, 0);
  GcsFilename xmlFilename = new GcsFilename(stagingBucket, prefix + ".xml.ghostryde");
  GcsFilename xmlLengthFilename = new GcsFilename(stagingBucket, prefix + ".xml.length");
  GcsFilename rydeFile = new GcsFilename(brdaBucket, prefix + ".ryde");
  GcsFilename sigFile = new GcsFilename(brdaBucket, prefix + ".sig");

  long xmlLength = readXmlLength(xmlLengthFilename);

  logger.atInfo().log("Writing %s and %s", rydeFile, sigFile);
  try (InputStream gcsInput = gcsUtils.openInputStream(xmlFilename);
      InputStream ghostrydeDecoder = Ghostryde.decoder(gcsInput, stagingDecryptionKey);
      OutputStream rydeOut = gcsUtils.openOutputStream(rydeFile);
      OutputStream sigOut = gcsUtils.openOutputStream(sigFile);
      RydeEncoder rydeEncoder = new RydeEncoder.Builder()
          .setRydeOutput(rydeOut, receiverKey)
          .setSignatureOutput(sigOut, signingKey)
          .setFileMetadata(prefix, xmlLength, watermark)
          .build()) {
    ByteStreams.copy(ghostrydeDecoder, rydeEncoder);
  }
}
 
Example 8
Source Project: appengine-gcs-client   Source File: OauthRawGcsService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public RawGcsCreationToken beginObjectCreation(
    GcsFilename filename, GcsFileOptions options, long timeoutMillis) throws IOException {
  HTTPRequest req = makeRequest(filename, null, POST, timeoutMillis);
  req.setHeader(RESUMABLE_HEADER);
  addOptionsHeaders(req, options);
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  if (resp.getResponseCode() == 201) {
    String location = URLFetchUtils.getSingleHeader(resp, LOCATION);
    String queryString = new URL(location).getQuery();
    Preconditions.checkState(
        queryString != null, LOCATION + " header," + location + ", witout a query string");
    Map<String, String> params = Splitter.on('&').withKeyValueSeparator('=').split(queryString);
    Preconditions.checkState(params.containsKey(UPLOAD_ID),
        LOCATION + " header," + location + ", has a query string without " + UPLOAD_ID);
    return new GcsRestCreationToken(filename, params.get(UPLOAD_ID), 0);
  } else {
    throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
}
 
Example 9
Source Project: appengine-gcs-client   Source File: OauthRawGcsService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void copyObject(GcsFilename source, GcsFilename dest, GcsFileOptions fileOptions,
    long timeoutMillis) throws IOException {
  HTTPRequest req = makeRequest(dest, null, PUT, timeoutMillis);
  req.setHeader(new HTTPHeader(X_GOOG_COPY_SOURCE, makePath(source)));
  if (fileOptions != null) {
    req.setHeader(REPLACE_METADATA_HEADER);
    addOptionsHeaders(req, fileOptions);
  }
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  if (resp.getResponseCode() != 200) {
    throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
}
 
Example 10
Source Project: nomulus   Source File: RdeReportAction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void runWithLock(DateTime watermark) throws Exception {
  Cursor cursor =
      ofy().load().key(Cursor.createKey(CursorType.RDE_UPLOAD, Registry.get(tld))).now();
  loadAndCompare(cursor, tld);
  DateTime cursorTime = getCursorTimeOrStartOfTime(cursor);
  if (isBeforeOrAt(cursorTime, watermark)) {
    throw new NoContentException(
        String.format(
            "Waiting on RdeUploadAction for TLD %s to send %s report; "
                + "last upload completion was at %s",
            tld, watermark, cursorTime));
  }
  String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, 0);
  GcsFilename reportFilename = new GcsFilename(bucket, prefix + "-report.xml.ghostryde");
  verify(gcsUtils.existsAndNotEmpty(reportFilename), "Missing file: %s", reportFilename);
  reporter.send(readReportFromGcs(reportFilename));
  response.setContentType(PLAIN_TEXT_UTF_8);
  response.setPayload(String.format("OK %s %s\n", tld, watermark));
}
 
Example 11
@Test
public void makeUrlShouldCorrectlyEncodeQueryString() {
  Map<String, String> queryStrings = new LinkedHashMap<>();
  queryStrings.put("upload_id", "} 20");
  queryStrings.put("composed", null);
  queryStrings.put("val=ue", "%7D&");
  queryStrings.put("regular", "val");
  queryStrings.put("k e+y", "=v a+lu&e=");
  String url = makeUrl(new GcsFilename(BUCKET, "object"), queryStrings).toString();
  String expected = URL_PREFIX + "object?"
      + "upload_id=%7D+20&"
      + "composed&"
      + "val%3Due=%257D%26&"
      + "regular=val&"
      + "k+e%2By=%3Dv+a%2Blu%26e%3D";
  assertEquals(expected, url);
}
 
Example 12
Source Project: appengine-gcs-client   Source File: OauthRawGcsService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void composeObject(Iterable<String> source, GcsFilename dest, long timeoutMillis)
    throws IOException {
  StringBuilder xmlContent = new StringBuilder(Iterables.size(source) * 50);
  Escaper escaper = XmlEscapers.xmlContentEscaper();
  xmlContent.append("<ComposeRequest>");
  for (String srcFileName : source) {
    xmlContent.append("<Component><Name>")
        .append(escaper.escape(srcFileName))
        .append("</Name></Component>");
  }
  xmlContent.append("</ComposeRequest>");
  HTTPRequest req = makeRequest(
      dest, COMPOSE_QUERY_STRINGS, PUT, timeoutMillis, xmlContent.toString().getBytes(UTF_8));
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  if (resp.getResponseCode() != 200) {
    throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
}
 
Example 13
Source Project: nomulus   Source File: IcannReportingUploadActionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSuccess_advancesCursor() throws Exception {
  writeGcsFile(
      gcsService,
      new GcsFilename("basin/icann/monthly/2006-06", "tld-activity-200606.csv"),
      PAYLOAD_SUCCESS);
  when(mockReporter.send(PAYLOAD_SUCCESS, "tld-activity-200606.csv")).thenReturn(true);
  IcannReportingUploadAction action = createAction();
  action.run();
  ofy().clearSessionCache();
  Cursor cursor =
      ofy()
          .load()
          .key(Cursor.createKey(CursorType.ICANN_UPLOAD_ACTIVITY, Registry.get("tld")))
          .now();
  assertThat(cursor.getCursorTime()).isEqualTo(DateTime.parse("2006-08-01TZ"));
}
 
Example 14
Source Project: appengine-gcs-client   Source File: LocalRawGcsService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void composeObject(Iterable<String> source, GcsFilename dest, long timeoutMillis)
    throws IOException {
  ensureInitialized();
  int size = Iterables.size(source);
  if (size > 32) {
    throw new IOException("Compose attempted with too many components. Limit is 32");
  }
  if (size < 2) {
    throw new IOException("You must provide at least two source components.");
  }
  Token token = beginObjectCreation(dest, GcsFileOptions.getDefaultInstance(), timeoutMillis);

  for (String filename : source) {
    GcsFilename sourceGcsFilename = new GcsFilename(dest.getBucketName(), filename);
    appendFileContentsToToken(sourceGcsFilename, token);
  }
  finishObjectCreation(token, ByteBuffer.allocate(0), timeoutMillis);
}
 
Example 15
Source Project: appengine-gcs-client   Source File: LocalRawGcsService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void copyObject(GcsFilename source, GcsFilename dest, GcsFileOptions fileOptions,
    long timeoutMillis) throws IOException {
  ensureInitialized();
  GcsFileMetadata meta = getObjectMetadata(source, timeoutMillis);
  if (meta == null) {
    throw new FileNotFoundException(this + ": No such file: " + source);
  }
  if (fileOptions == null) {
    fileOptions = meta.getOptions();
  }

  Token token = beginObjectCreation(dest, fileOptions, timeoutMillis);
  appendFileContentsToToken(source, token);
  finishObjectCreation(token, ByteBuffer.allocate(0), timeoutMillis);
}
 
Example 16
Source Project: nomulus   Source File: CopyDetailReportsActionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSuccess_nonDetailReportFiles_notSent() throws IOException{
  writeGcsFile(
      gcsService,
      new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
      "hola,mundo\n3,4".getBytes(UTF_8));

  writeGcsFile(
      gcsService,
      new GcsFilename("test-bucket", "results/not_a_detail_report_2017-10_TheRegistrar_test.csv"),
      "hello,world\n1,2".getBytes(UTF_8));
  action.run();
  verify(driveConnection)
      .createOrUpdateFile(
          "invoice_details_2017-10_TheRegistrar_hello.csv",
          MediaType.CSV_UTF_8,
          "0B-12345",
          "hola,mundo\n3,4".getBytes(UTF_8));
  // Verify we didn't copy the non-detail report file.
  verifyNoMoreInteractions(driveConnection);
  assertThat(response.getStatus()).isEqualTo(SC_OK);
  assertThat(response.getContentType()).isEqualTo(MediaType.PLAIN_TEXT_UTF_8);
  assertThat(response.getPayload()).isEqualTo("Copied detail reports.\n");
}
 
Example 17
Source Project: nomulus   Source File: CopyDetailReportsActionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSuccess_transientIOException_retries() throws IOException {
  writeGcsFile(
      gcsService,
      new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
      "hola,mundo\n3,4".getBytes(UTF_8));
  when(driveConnection.createOrUpdateFile(any(), any(), any(), any()))
      .thenThrow(new IOException("expected"))
      .thenReturn("success");

  action.run();
  verify(driveConnection, times(2))
      .createOrUpdateFile(
          "invoice_details_2017-10_TheRegistrar_hello.csv",
          MediaType.CSV_UTF_8,
          "0B-12345",
          "hola,mundo\n3,4".getBytes(UTF_8));
  assertThat(response.getStatus()).isEqualTo(SC_OK);
  assertThat(response.getContentType()).isEqualTo(MediaType.PLAIN_TEXT_UTF_8);
  assertThat(response.getPayload()).isEqualTo("Copied detail reports.\n");
}
 
Example 18
Source Project: nomulus   Source File: RdeStagingActionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMapReduce_idnTables_goInDeposit() throws Exception {
  clock.setTo(DateTime.parse("1999-12-31TZ"));
  createTldWithEscrowEnabled("fop");
  makeDomainBase(clock, "fop");

  clock.setTo(DateTime.parse("2000-01-01TZ"));
  action.run();
  executeTasksUntilEmpty("mapreduce", clock);

  GcsFilename filename = new GcsFilename("rde-bucket", "fop_2000-01-01_full_S1_R0.xml.ghostryde");
  XjcRdeDeposit deposit =
      unmarshal(
          XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, filename), decryptKey));
  XjcRdeDomain domain = extractAndRemoveContentWithType(XjcRdeDomain.class, deposit);
  XjcRdeIdn firstIdn = extractAndRemoveContentWithType(XjcRdeIdn.class, deposit);
  XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit);

  assertThat(domain.getIdnTableId()).isEqualTo("extended_latin");
  assertThat(firstIdn.getId()).isEqualTo("extended_latin");
  assertThat(firstIdn.getUrl()).isEqualTo(EXTENDED_LATIN.getTable().getUrl().toString());
  assertThat(firstIdn.getUrlPolicy()).isEqualTo(EXTENDED_LATIN.getTable().getPolicy().toString());
  assertThat(mapifyCounts(header))
      .containsEntry(RdeResourceType.IDN.getUri(), (long) IdnTableEnum.values().length);
}
 
Example 19
Source Project: appengine-gcs-client   Source File: OauthRawGcsService.java    License: Apache License 2.0 6 votes vote down vote up
/** True if deleted, false if not found. */
@Override
public boolean deleteObject(GcsFilename filename, long timeoutMillis) throws IOException {
  HTTPRequest req = makeRequest(filename, null, DELETE, timeoutMillis);
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  switch (resp.getResponseCode()) {
    case 204:
      return true;
    case 404:
      return false;
    default:
      throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
}
 
Example 20
Source Project: nomulus   Source File: RdeUploadActionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRunWithLock_copiesOnGcs() throws Exception {
  int port = sftpd.serve("user", "password", folder.getRoot());
  URI uploadUrl = URI.create(String.format("sftp://user:[email protected]:%d/", port));
  DateTime stagingCursor = DateTime.parse("2010-10-18TZ");
  DateTime uploadCursor = DateTime.parse("2010-10-17TZ");
  persistResource(Cursor.create(RDE_STAGING, stagingCursor, Registry.get("tld")));
  createAction(uploadUrl).runWithLock(uploadCursor);
  assertThat(response.getStatus()).isEqualTo(200);
  assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
  assertThat(response.getPayload()).isEqualTo("OK tld 2010-10-17T00:00:00.000Z\n");
  assertNoTasksEnqueued("rde-upload");
  // Assert that both files are written to SFTP and GCS, and that the contents are identical.
  String rydeFilename = "tld_2010-10-17_full_S1_R0.ryde";
  String sigFilename = "tld_2010-10-17_full_S1_R0.sig";
  assertThat(folder.getRoot().list()).asList().containsExactly(rydeFilename, sigFilename);
  assertThat(readGcsFile(gcsService, new GcsFilename("bucket", rydeFilename)))
      .isEqualTo(Files.toByteArray(new File(folder.getRoot(), rydeFilename)));
  assertThat(readGcsFile(gcsService, new GcsFilename("bucket", sigFilename)))
      .isEqualTo(Files.toByteArray(new File(folder.getRoot(), sigFilename)));
}
 
Example 21
Source Project: appengine-gcs-client   Source File: OauthRawGcsService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public GcsFileMetadata getObjectMetadata(GcsFilename filename, long timeoutMillis)
    throws IOException {
  HTTPRequest req = makeRequest(filename, null, HEAD, timeoutMillis);
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  int responseCode = resp.getResponseCode();
  if (responseCode == 404) {
    return null;
  }
  if (responseCode != 200) {
    throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
  return getMetadataFromResponse(
      filename, resp, getLengthFromHeader(resp, X_GOOG_CONTENT_LENGTH));
}
 
Example 22
Source Project: nomulus   Source File: RestoreCommitLogsActionTest.java    License: Apache License 2.0 6 votes vote down vote up
private Iterable<ImmutableObject> saveDiffFile(
    CommitLogCheckpoint checkpoint, ImmutableObject... entities) throws IOException {
  DateTime now = checkpoint.getCheckpointTime();
  List<ImmutableObject> allEntities = Lists.asList(checkpoint, entities);
  ByteArrayOutputStream output = new ByteArrayOutputStream();
  for (ImmutableObject entity : allEntities) {
    serializeEntity(entity, output);
  }
  gcsService.createOrReplace(
      new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + now),
      new GcsFileOptions.Builder()
          .addUserMetadata(LOWER_BOUND_CHECKPOINT, now.minusMinutes(1).toString())
          .build(),
      ByteBuffer.wrap(output.toByteArray()));
  return allEntities;
}
 
Example 23
Source Project: nomulus   Source File: ExportDomainListsActionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void test_outputsOnlyDomainsOnRealTlds() throws Exception {
  persistActiveDomain("onetwo.tld");
  persistActiveDomain("rudnitzky.tld");
  persistActiveDomain("wontgo.testtld");
  runMapreduce();
  GcsFilename existingFile = new GcsFilename("outputbucket", "tld.txt");
  String tlds = new String(readGcsFile(gcsService, existingFile), UTF_8).trim();
  // Check that it only contains the domains on the real TLD, and not the test one.
  assertThat(tlds).isEqualTo("onetwo.tld\nrudnitzky.tld");
  // Make sure that the test TLD file wasn't written out.
  GcsFilename nonexistentFile = new GcsFilename("outputbucket", "testtld.txt");
  assertThrows(FileNotFoundException.class, () -> readGcsFile(gcsService, nonexistentFile));
  ListResult ls = gcsService.list("outputbucket", ListOptions.DEFAULT);
  assertThat(ls.next().getName()).isEqualTo("tld.txt");
  // Make sure that no other files were written out.
  assertThat(ls.hasNext()).isFalse();
  verifyExportedToDrive("brouhaha", "onetwo.tld\nrudnitzky.tld");
  verifyNoMoreInteractions(driveConnection);
}
 
Example 24
Source Project: appengine-gcs-client   Source File: LocalRawGcsService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public GcsFileMetadata getObjectMetadata(GcsFilename filename, long timeoutMillis)
    throws IOException {
  ensureInitialized();
  Entity entity;
  try {
    entity = datastore.get(null, makeKey(filename));
    return createGcsFileMetadata(entity, filename);
  } catch (EntityNotFoundException ex1) {
    try {
      entity = datastore.get(null, makeBlobstoreKey(filename));
      return createGcsFileMetadataFromBlobstore(entity, filename);
    } catch (EntityNotFoundException ex2) {
      return null;
    }
  }
}
 
Example 25
Source Project: appengine-gcs-client   Source File: OauthRawGcsService.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
HTTPRequest makeRequest(GcsFilename filename, @Nullable Map<String, String> queryStrings,
    HTTPMethod method, long timeoutMillis, byte[] payload) {
  HTTPRequest request = new HTTPRequest(makeUrl(filename, queryStrings), method,
      FetchOptions.Builder.disallowTruncate()
          .doNotFollowRedirects()
          .validateCertificate()
          .setDeadline(timeoutMillis / 1000.0));
  for (HTTPHeader header : headers) {
    request.addHeader(header);
  }
  request.addHeader(USER_AGENT);
  if (payload != null && payload.length > 0) {
    request.setHeader(new HTTPHeader(CONTENT_LENGTH, String.valueOf(payload.length)));
    try {
      request.setHeader(new HTTPHeader(CONTENT_MD5,
          BaseEncoding.base64().encode(MessageDigest.getInstance("MD5").digest(payload))));
    } catch (NoSuchAlgorithmException e) {
      log.severe(
          "Unable to get a MessageDigest instance, no Content-MD5 header sent.\n" + e.toString());
    }
    request.setPayload(payload);
  } else {
    request.setHeader(ZERO_CONTENT_LENGTH);
  }
  return request;
}
 
Example 26
Source Project: appinventor-extensions   Source File: ObjectifyStorageIo.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void deleteTempFile(String fileName) throws IOException {
  if (!fileName.startsWith("__TEMP__")) {
    throw new RuntimeException("deleteTempFile (" + fileName + ") Invalid File Name");
  }
  gcsService.delete(new GcsFilename(GCS_BUCKET_NAME, fileName));
}
 
Example 27
Source Project: appinventor-extensions   Source File: ObjectifyStorageIo.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
void setGcsFileContent(String gcsPath, byte[] content) throws IOException {
  GcsOutputChannel outputChannel = gcsService.createOrReplace(
      new GcsFilename(GCS_BUCKET_NAME, gcsPath),
      GcsFileOptions.getDefaultInstance());
  outputChannel.write(ByteBuffer.wrap(content));
  outputChannel.close();
}
 
Example 28
Source Project: nomulus   Source File: IcannReportingUploadAction.java    License: Apache License 2.0 5 votes vote down vote up
private void verifyFileExists(GcsFilename gcsFilename) {
  checkArgument(
      gcsUtils.existsAndNotEmpty(gcsFilename),
      "Object %s in bucket %s not found",
      gcsFilename.getObjectName(),
      gcsFilename.getBucketName());
}
 
Example 29
private ImmutableSet<RegistrarThreatMatches> getFromFile(GcsFilename spec11ReportFilename)
    throws IOException, JSONException {
  ImmutableSet.Builder<RegistrarThreatMatches> builder = ImmutableSet.builder();
  try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename)) {
    ImmutableList<String> reportLines =
        ImmutableList.copyOf(CharStreams.toString(new InputStreamReader(in, UTF_8)).split("\n"));
    // Iterate from 1 to size() to skip the header at line 0.
    for (int i = 1; i < reportLines.size(); i++) {
      builder.add(parseRegistrarThreatMatch(reportLines.get(i)));
    }
    return builder.build();
  }
}
 
Example 30
Source Project: nomulus   Source File: BillingEmailUtils.java    License: Apache License 2.0 5 votes vote down vote up
/** Sends an e-mail to all expected recipients with an attached overall invoice from GCS. */
void emailOverallInvoice() {
  try {
    String invoiceFile = String.format("%s-%s.csv", invoiceFilePrefix, yearMonth);
    GcsFilename invoiceFilename =
        new GcsFilename(billingBucket, invoiceDirectoryPrefix + invoiceFile);
    try (InputStream in = gcsUtils.openInputStream(invoiceFilename)) {
      emailService.sendEmail(
          EmailMessage.newBuilder()
              .setSubject(String.format("Domain Registry invoice data %s", yearMonth))
              .setBody(
                  String.format("Attached is the %s invoice for the domain registry.", yearMonth))
              .setFrom(outgoingEmailAddress)
              .setRecipients(invoiceEmailRecipients)
              .setAttachment(
                  Attachment.newBuilder()
                      .setContent(CharStreams.toString(new InputStreamReader(in, UTF_8)))
                      .setContentType(MediaType.CSV_UTF_8)
                      .setFilename(invoiceFile)
                      .build())
              .build());
    }
  } catch (Throwable e) {
    // Strip one layer, because callWithRetry wraps in a RuntimeException
    sendAlertEmail(
        String.format("Emailing invoice failed due to %s", getRootCause(e).getMessage()));
    throw new RuntimeException("Emailing invoice failed", e);
  }
}