com.google.appengine.tools.cloudstorage.GcsFileMetadata Java Examples

The following examples show how to use com.google.appengine.tools.cloudstorage.GcsFileMetadata. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LocalRawGcsService.java    From appengine-gcs-client with Apache License 2.0 6 votes vote down vote up
@Override
public GcsFileMetadata getObjectMetadata(GcsFilename filename, long timeoutMillis)
    throws IOException {
  ensureInitialized();
  Entity entity;
  try {
    entity = datastore.get(null, makeKey(filename));
    return createGcsFileMetadata(entity, filename);
  } catch (EntityNotFoundException ex1) {
    try {
      entity = datastore.get(null, makeBlobstoreKey(filename));
      return createGcsFileMetadataFromBlobstore(entity, filename);
    } catch (EntityNotFoundException ex2) {
      return null;
    }
  }
}
 
Example #2
Source File: LocalRawGcsService.java    From appengine-gcs-client with Apache License 2.0 6 votes vote down vote up
@Override
public void copyObject(GcsFilename source, GcsFilename dest, GcsFileOptions fileOptions,
    long timeoutMillis) throws IOException {
  ensureInitialized();
  GcsFileMetadata meta = getObjectMetadata(source, timeoutMillis);
  if (meta == null) {
    throw new FileNotFoundException(this + ": No such file: " + source);
  }
  if (fileOptions == null) {
    fileOptions = meta.getOptions();
  }

  Token token = beginObjectCreation(dest, fileOptions, timeoutMillis);
  appendFileContentsToToken(source, token);
  finishObjectCreation(token, ByteBuffer.allocate(0), timeoutMillis);
}
 
Example #3
Source File: OauthRawGcsService.java    From appengine-gcs-client with Apache License 2.0 6 votes vote down vote up
@Override
public GcsFileMetadata getObjectMetadata(GcsFilename filename, long timeoutMillis)
    throws IOException {
  HTTPRequest req = makeRequest(filename, null, HEAD, timeoutMillis);
  HTTPResponse resp;
  try {
    resp = urlfetch.fetch(req);
  } catch (IOException e) {
    throw createIOException(new HTTPRequestInfo(req), e);
  }
  int responseCode = resp.getResponseCode();
  if (responseCode == 404) {
    return null;
  }
  if (responseCode != 200) {
    throw HttpErrorHandler.error(new HTTPRequestInfo(req), resp);
  }
  return getMetadataFromResponse(
      filename, resp, getLengthFromHeader(resp, X_GOOG_CONTENT_LENGTH));
}
 
Example #4
Source File: GcsUtils.java    From nomulus with Apache License 2.0 5 votes vote down vote up
/** Returns {@code true} if a file exists and is non-empty on Google Cloud Storage. */
public boolean existsAndNotEmpty(GcsFilename file) {
  GcsFileMetadata metadata;
  try {
    metadata = gcsService.getMetadata(file);
  } catch (IOException e) {
    logger.atWarning().withCause(e).log("Failed to check if GCS file exists");
    return false;
  }
  return metadata != null && metadata.getLength() > 0;
}
 
Example #5
Source File: GcsDiffFileLister.java    From nomulus with Apache License 2.0 5 votes vote down vote up
/**
 * Traverses the sequence of diff files backwards from checkpointTime and inserts the file
 * metadata into "sequence".  Returns true if a complete sequence was discovered, false if one or
 * more files are missing.
 */
private boolean constructDiffSequence(
    Map<DateTime, ListenableFuture<GcsFileMetadata>> upperBoundTimesToMetadata,
    DateTime fromTime,
    DateTime lastTime,
    TreeMap<DateTime, GcsFileMetadata> sequence) {
  DateTime checkpointTime = lastTime;
  while (isBeforeOrAt(fromTime, checkpointTime)) {
    GcsFileMetadata metadata;
    if (upperBoundTimesToMetadata.containsKey(checkpointTime)) {
      metadata = Futures.getUnchecked(upperBoundTimesToMetadata.get(checkpointTime));
    } else {
      String filename = DIFF_FILE_PREFIX + checkpointTime;
      logger.atInfo().log("Patching GCS list; discovered file: %s", filename);
      metadata = getMetadata(filename);

      // If we hit a gap, quit.
      if (metadata == null) {
        logger.atInfo().log(
            "Gap discovered in sequence terminating at %s, missing file: %s",
            sequence.lastKey(), filename);
        logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
        return false;
      }
    }
    sequence.put(checkpointTime, metadata);
    checkpointTime = getLowerBoundTime(metadata);
  }
  logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
  return true;
}
 
Example #6
Source File: GcsDiffFileLister.java    From nomulus with Apache License 2.0 5 votes vote down vote up
private GcsFileMetadata getMetadata(String filename) {
  try {
    return gcsService.getMetadata(new GcsFilename(gcsBucket, filename));
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example #7
Source File: GcsDiffFileListerTest.java    From nomulus with Apache License 2.0 5 votes vote down vote up
private Iterable<DateTime> extractTimesFromDiffFiles(List<GcsFileMetadata> diffFiles) {
  return transform(
      diffFiles,
      metadata ->
          DateTime.parse(
              metadata.getFilename().getObjectName().substring(DIFF_FILE_PREFIX.length())));
}
 
Example #8
Source File: LocalRawGcsService.java    From appengine-gcs-client with Apache License 2.0 5 votes vote down vote up
private GcsFileMetadata createGcsFileMetadataFromBlobstore(Entity entity, GcsFilename filename) {
  return new GcsFileMetadata(
      filename,
      GcsFileOptions.getDefaultInstance(),
      "",
      (Long) entity.getProperty("size"),
      (Date) entity.getProperty("creation"));
}
 
Example #9
Source File: GCSClientTest.java    From appengine-tck with Apache License 2.0 5 votes vote down vote up
@Test
@InSequence(4)
public void testOptionsAndMetadata() throws IOException {
    GcsFilename filename = new GcsFilename(bucket, OBJECT_NAME + "4");
    GcsFileOptions option = new GcsFileOptions.Builder()
        .acl("public-read")
        .cacheControl("Cache-Control: public, max-age=3600")
        .contentEncoding("Content-Encoding: gzip")
        .contentDisposition("Content-Disposition: attachment")
        .mimeType("text/html")
        .addUserMetadata("userKey", "UserMetadata")
        .build();

    GcsOutputChannel writeChannel = gcsService.createOrReplace(filename, option);
    try (PrintWriter out = new PrintWriter(Channels.newWriter(writeChannel, "UTF8"))) {
        out.println(CONTENT);
        out.flush();
    }

    GcsFileMetadata metaData = gcsService.getMetadata(filename);
    GcsFileOptions option2 = metaData.getOptions();
    try {
        assertEquals(filename, metaData.getFilename());
    } finally {
        gcsService.delete(filename);
    }

    assertEquals("Cache-Control: public, max-age=3600", option2.getCacheControl());
    assertEquals("Content-Encoding: gzip", option2.getContentEncoding());
    assertEquals("Content-Disposition: attachment", option2.getContentDisposition());
    assertEquals("text/html", option2.getMimeType());
    assertEquals("Content-Encoding: gzip", option2.getContentEncoding());
    Map<String, String> userMetadata = option2.getUserMetadata();
    assertEquals(1, userMetadata.size());
    String key = userMetadata.keySet().iterator().next();
    assertEquals("UserMetadata", userMetadata.get(key));
    assertEquals("public-read", option2.getAcl());
}
 
Example #10
Source File: GcsDiffFileLister.java    From nomulus with Apache License 2.0 4 votes vote down vote up
ImmutableList<GcsFileMetadata> listDiffFiles(DateTime fromTime, @Nullable DateTime toTime) {
  logger.atInfo().log("Requested restore from time: %s", fromTime);
  if (toTime != null) {
    logger.atInfo().log("  Until time: %s", toTime);
  }
  // List all of the diff files on GCS and build a map from each file's upper checkpoint time
  // (extracted from the filename) to its asynchronously-loaded metadata, keeping only files with
  // an upper checkpoint time > fromTime.
  TreeMap<DateTime, ListenableFuture<GcsFileMetadata>> upperBoundTimesToMetadata
      = new TreeMap<>();
  Iterator<ListItem> listItems;
  try {
    // TODO(b/23554360): Use a smarter prefixing strategy to speed this up.
    listItems = gcsService.list(
        gcsBucket,
        new ListOptions.Builder().setPrefix(DIFF_FILE_PREFIX).build());
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
  DateTime lastUpperBoundTime = START_OF_TIME;
  while (listItems.hasNext()) {
    final String filename = listItems.next().getName();
    DateTime upperBoundTime = DateTime.parse(filename.substring(DIFF_FILE_PREFIX.length()));
    if (isInRange(upperBoundTime, fromTime, toTime)) {
      upperBoundTimesToMetadata.put(upperBoundTime, executor.submit(() -> getMetadata(filename)));
      lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime);
    }
  }
  if (upperBoundTimesToMetadata.isEmpty()) {
    logger.atInfo().log("No files found");
    return ImmutableList.of();
  }

  // Reconstruct the sequence of files by traversing backwards from "lastUpperBoundTime" (i.e. the
  // last file that we found) and finding its previous file until we either run out of files or
  // get to one that preceeds "fromTime".
  //
  // GCS file listing is eventually consistent, so it's possible that we are missing a file. The
  // metadata of a file is sufficient to identify the preceding file, so if we start from the
  // last file and work backwards we can verify that we have no holes in our chain (although we
  // may be missing files at the end).
  TreeMap<DateTime, GcsFileMetadata> sequence = new TreeMap<>();
  logger.atInfo().log("Restoring until: %s", lastUpperBoundTime);
  boolean inconsistentFileSet = !constructDiffSequence(
      upperBoundTimesToMetadata, fromTime, lastUpperBoundTime, sequence);

  // Verify that all of the elements in the original set are represented in the sequence.  If we
  // find anything that's not represented, construct a sequence for it.
  boolean checkForMoreExtraDiffs = true;  // Always loop at least once.
  while (checkForMoreExtraDiffs) {
    checkForMoreExtraDiffs = false;
    for (DateTime key : upperBoundTimesToMetadata.descendingKeySet()) {
      if (!isInRange(key, fromTime, toTime)) {
        break;
      }
      if (!sequence.containsKey(key)) {
        constructDiffSequence(upperBoundTimesToMetadata, fromTime, key, sequence);
        checkForMoreExtraDiffs = true;
        inconsistentFileSet = true;
        break;
      }
    }
  }

  checkState(
      !inconsistentFileSet,
      "Unable to compute commit diff history, there are either gaps or forks in the history "
      + "file set.  Check log for details.");

  logger.atInfo().log(
      "Actual restore from time: %s", getLowerBoundTime(sequence.firstEntry().getValue()));
  logger.atInfo().log("Found %d files to restore", sequence.size());
  return ImmutableList.copyOf(sequence.values());
}
 
Example #11
Source File: GcsDiffFileLister.java    From nomulus with Apache License 2.0 4 votes vote down vote up
private DateTime getLowerBoundTime(GcsFileMetadata metadata) {
  return DateTime.parse(metadata.getOptions().getUserMetadata().get(LOWER_BOUND_CHECKPOINT));
}
 
Example #12
Source File: RestoreCommitLogsAction.java    From nomulus with Apache License 2.0 4 votes vote down vote up
@Override
public void run() {
  checkArgument(
      RegistryEnvironment.get() == RegistryEnvironment.ALPHA
          || RegistryEnvironment.get() == RegistryEnvironment.CRASH
          || RegistryEnvironment.get() == RegistryEnvironment.UNITTEST,
      "DO NOT RUN ANYWHERE ELSE EXCEPT ALPHA, CRASH OR TESTS.");
  if (dryRun) {
    logger.atInfo().log("Running in dryRun mode");
  }
  List<GcsFileMetadata> diffFiles = diffLister.listDiffFiles(fromTime, toTime);
  if (diffFiles.isEmpty()) {
    logger.atInfo().log("Nothing to restore");
    return;
  }
  Map<Integer, DateTime> bucketTimestamps = new HashMap<>();
  CommitLogCheckpoint lastCheckpoint = null;
  for (GcsFileMetadata metadata : diffFiles) {
    logger.atInfo().log("Restoring: %s", metadata.getFilename().getObjectName());
    try (InputStream input = Channels.newInputStream(
        gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
      PeekingIterator<ImmutableObject> commitLogs =
          peekingIterator(createDeserializingIterator(input));
      lastCheckpoint = (CommitLogCheckpoint) commitLogs.next();
      saveOfy(ImmutableList.of(lastCheckpoint));  // Save the checkpoint itself.
      while (commitLogs.hasNext()) {
        CommitLogManifest manifest = restoreOneTransaction(commitLogs);
        bucketTimestamps.put(manifest.getBucketId(), manifest.getCommitTime());
      }
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }
  // Restore the CommitLogCheckpointRoot and CommitLogBuckets.
  saveOfy(
      Streams.concat(
              bucketTimestamps
                  .entrySet()
                  .stream()
                  .map(
                      entry ->
                          new CommitLogBucket.Builder()
                              .setBucketNum(entry.getKey())
                              .setLastWrittenTime(entry.getValue())
                              .build()),
              Stream.of(CommitLogCheckpointRoot.create(lastCheckpoint.getCheckpointTime())))
          .collect(toImmutableList()));
  logger.atInfo().log("Restore complete");
}
 
Example #13
Source File: LocalRawGcsService.java    From appengine-gcs-client with Apache License 2.0 4 votes vote down vote up
@Override
public Future<GcsFileMetadata> readObjectAsync(
    ByteBuffer dst, GcsFilename filename, long offset, long timeoutMillis) {
  Preconditions.checkArgument(offset >= 0, "%s: offset must be non-negative: %s", this, offset);
  try {
    ensureInitialized();
    GcsFileMetadata meta = getObjectMetadata(filename, timeoutMillis);
    if (meta == null) {
      return Futures.immediateFailedFuture(
          new FileNotFoundException(this + ": No such file: " + filename));
    }
    if (offset >= meta.getLength()) {
      return Futures.immediateFailedFuture(new BadRangeException(
          "The requested range cannot be satisfied. bytes=" + Long.toString(offset) + "-"
          + Long.toString(offset + dst.remaining()) + " the file is only " + meta.getLength()));
    }

    int read = 0;

    try (ReadableByteChannel readChannel = Channels.newChannel(
        blobStorage.fetchBlob(getBlobKeyForFilename(filename)))) {
      if (offset > 0) {
        long bytesRemaining = offset;
        ByteBuffer seekBuffer = ByteBuffer.allocate(1024);
        while (bytesRemaining > 0) {
          if (bytesRemaining < seekBuffer.limit()) {
            seekBuffer.limit((int) bytesRemaining);
          }
          read = readChannel.read(seekBuffer);
          if (read == -1) {
            return Futures.immediateFailedFuture(new BadRangeException(
                "The requested range cannot be satisfied; seek failed with "
                + Long.toString(bytesRemaining) + " bytes remaining."));

          }
          bytesRemaining -= read;
          seekBuffer.clear();
        }
        read = 0;
      }
      while (read != -1 && dst.hasRemaining()) {
        read = readChannel.read(dst);
      }
    }

    return Futures.immediateFuture(meta);
  } catch (IOException e) {
    return Futures.immediateFailedFuture(e);
  }
}
 
Example #14
Source File: LocalRawGcsService.java    From appengine-gcs-client with Apache License 2.0 4 votes vote down vote up
@Override
public ListItemBatch list(String bucket, String prefix, String delimiter,
    String marker, int maxResults, long timeoutMillis) throws IOException {
  ensureInitialized();
  Query query = makeQuery(bucket);
  int prefixLength;
  if (!Strings.isNullOrEmpty(prefix)) {
    Key keyPrefix = makeKey(bucket, prefix);
    query.setFilter(new FilterPredicate(KEY_RESERVED_PROPERTY, GREATER_THAN_OR_EQUAL, keyPrefix));
    prefixLength = prefix.length();
  } else {
    prefixLength = 0;
  }
  FetchOptions fetchOptions = FetchOptions.Builder.withDefaults();
  if (marker != null) {
    fetchOptions.startCursor(Cursor.fromWebSafeString(marker));
  }
  List<ListItem> items = new ArrayList<>(maxResults);
  Set<String> prefixes = new HashSet<>();
  QueryResultIterator<Entity> dsResults =
      datastore.prepare(query).asQueryResultIterator(fetchOptions);
  while (items.size() < maxResults && dsResults.hasNext()) {
    Entity entity = dsResults.next();
    String name = entity.getKey().getName();
    if (prefixLength > 0 && !name.startsWith(prefix)) {
      break;
    }
    if (!Strings.isNullOrEmpty(delimiter)) {
      int delimiterIdx = name.indexOf(delimiter, prefixLength);
      if (delimiterIdx > 0) {
        name = name.substring(0, delimiterIdx + 1);
        if (prefixes.add(name)) {
          items.add(new ListItem.Builder().setName(name).setDirectory(true).build());
        }
        continue;
      }
    }
    GcsFilename filename = new GcsFilename(bucket, name);
    GcsFileMetadata metadata = createGcsFileMetadata(entity, filename);
    ListItem listItem = new ListItem.Builder()
        .setName(name)
        .setLength(metadata.getLength())
        .setLastModified(metadata.getLastModified())
        .build();
    items.add(listItem);
  }
  Cursor cursor = dsResults.getCursor();
  String nextMarker = null;
  if (items.size() == maxResults && cursor != null) {
    nextMarker = cursor.toWebSafeString();
  }
  return new ListItemBatch(items, nextMarker);
}
 
Example #15
Source File: OauthRawGcsService.java    From appengine-gcs-client with Apache License 2.0 4 votes vote down vote up
/**
 * Might not fill all of dst.
 */
@Override
public Future<GcsFileMetadata> readObjectAsync(final ByteBuffer dst, final GcsFilename filename,
    long startOffsetBytes, long timeoutMillis) {
  Preconditions.checkArgument(startOffsetBytes >= 0, "%s: offset must be non-negative: %s", this,
      startOffsetBytes);
  final int n = dst.remaining();
  Preconditions.checkArgument(n > 0, "%s: dst full: %s", this, dst);
  final int want = Math.min(READ_LIMIT_BYTES, n);

  final HTTPRequest req = makeRequest(filename, null, GET, timeoutMillis);
  req.setHeader(
      new HTTPHeader(RANGE, "bytes=" + startOffsetBytes + "-" + (startOffsetBytes + want - 1)));
  final HTTPRequestInfo info = new HTTPRequestInfo(req);
  return new FutureWrapper<HTTPResponse, GcsFileMetadata>(urlfetch.fetchAsync(req)) {
    @Override
    protected GcsFileMetadata wrap(HTTPResponse resp) throws IOException {
      long totalLength;
      switch (resp.getResponseCode()) {
        case 200:
          totalLength = getLengthFromHeader(resp, X_GOOG_CONTENT_LENGTH);
          break;
        case 206:
          totalLength = getLengthFromContentRange(resp);
          break;
        case 404:
          throw new FileNotFoundException("Could not find: " + filename);
        case 416:
          throw new BadRangeException("Requested Range not satisfiable; perhaps read past EOF? "
              + URLFetchUtils.describeRequestAndResponse(info, resp));
        default:
          throw HttpErrorHandler.error(info, resp);
      }
      byte[] content = resp.getContent();
      Preconditions.checkState(content.length <= want, "%s: got %s > wanted %s", this,
          content.length, want);
      dst.put(content);
      return getMetadataFromResponse(filename, resp, totalLength);
    }

    @Override
    protected Throwable convertException(Throwable e) {
      return OauthRawGcsService.convertException(info, e);
    }
  };
}