com.google.appengine.tools.cloudstorage.RetryParams Java Examples

The following examples show how to use com.google.appengine.tools.cloudstorage.RetryParams. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ObjectifyStorageIo.java    From appinventor-extensions with Apache License 2.0 6 votes vote down vote up
ObjectifyStorageIo() {
  RetryParams retryParams = new RetryParams.Builder().initialRetryDelayMillis(100)
    .retryMaxAttempts(10)
    .totalRetryPeriodMillis(10000).build();
  if (DEBUG) {
    LOG.log(Level.INFO, "RetryParams: getInitialRetryDelayMillis() = " + retryParams.getInitialRetryDelayMillis());
    LOG.log(Level.INFO, "RetryParams: getRequestTimeoutMillis() = " + retryParams.getRequestTimeoutMillis());
    LOG.log(Level.INFO, "RetryParams: getRetryDelayBackoffFactor() = " + retryParams.getRetryDelayBackoffFactor());
    LOG.log(Level.INFO, "RetryParams: getRetryMaxAttempts() = " + retryParams.getRetryMaxAttempts());
    LOG.log(Level.INFO, "RetryParams: getRetryMinAttempts() = " + retryParams.getRetryMinAttempts());
    LOG.log(Level.INFO, "RetryParams: getTotalRetryPeriodMillis() = " + retryParams.getTotalRetryPeriodMillis());
  }
  gcsService = GcsServiceFactory.createGcsService(retryParams);
  memcache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
  initMotd();
}
 
Example #2
Source File: GenerateZoneFilesAction.java    From nomulus with Apache License 2.0 6 votes vote down vote up
@Override
public void reduce(String tld, ReducerInput<String> stanzas) {
  String stanzaCounter = tld + " stanzas";
  GcsFilename filename =
      new GcsFilename(bucket, String.format(FILENAME_FORMAT, tld, exportTime));
  GcsUtils cloudStorage =
      new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
  try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename);
      Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8);
      PrintWriter writer = new PrintWriter(osWriter)) {
    writer.printf(HEADER_FORMAT, tld);
    for (Iterator<String> stanzaIter = filter(stanzas, Objects::nonNull);
        stanzaIter.hasNext(); ) {
      writer.println(stanzaIter.next());
      getContext().incrementCounter(stanzaCounter);
    }
    writer.flush();
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example #3
Source File: ExportDomainListsAction.java    From nomulus with Apache License 2.0 5 votes vote down vote up
private void exportToGcs(String tld, String domains) {
  GcsFilename filename = new GcsFilename(gcsBucket, tld + ".txt");
  GcsUtils cloudStorage =
      new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
  try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename);
      Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8)) {
    osWriter.write(domains);
  } catch (IOException e) {
    logger.atSevere().withCause(e).log(
        "Error exporting registered domains for TLD %s to GCS.", tld);
  }
  getContext().incrementCounter("domain lists written out to GCS");
}
 
Example #4
Source File: AppEngineTaskQueue.java    From appengine-pipelines with Apache License 2.0 5 votes vote down vote up
private TaskOptions toTaskOptions(Task task) {
  final QueueSettings queueSettings = task.getQueueSettings();

  TaskOptions taskOptions = TaskOptions.Builder.withUrl(TaskHandler.handleTaskUrl());
  if (queueSettings.getOnBackend() != null) {
    taskOptions.header("Host", BackendServiceFactory.getBackendService().getBackendAddress(
        queueSettings.getOnBackend()));
  } else {

    String versionHostname = RetryHelper.runWithRetries(new Callable<String>() {
      @Override
      public String call() {
        ModulesService service = ModulesServiceFactory.getModulesService();
        String module = queueSettings.getOnModule();
        String version = queueSettings.getModuleVersion();
        if (module == null) {
          module = service.getCurrentModule();
          version = service.getCurrentVersion();
        }
        return service.getVersionHostname(module, version);
      }
    }, RetryParams.getDefaultInstance(), MODULES_EXCEPTION_HANDLER);
    taskOptions.header("Host", versionHostname);
  }

  Long delayInSeconds = queueSettings.getDelayInSeconds();
  if (null != delayInSeconds) {
    taskOptions.countdownMillis(delayInSeconds * 1000L);
    queueSettings.setDelayInSeconds(null);
  }
  addProperties(taskOptions, task.toProperties());
  String taskName = task.getName();
  if (null != taskName) {
    taskOptions.taskName(taskName);
  }
  return taskOptions;
}
 
Example #5
Source File: GoogleCloudStorageHelper.java    From teammates with GNU General Public License v2.0 5 votes vote down vote up
/**
 * Writes a byte array {@code imageData} as image to the Google Cloud Storage,
 * with the {@code googleId} as the identifier name for the image.
 *
 * @return the {@link BlobKey} used as the image's identifier in Google Cloud Storage
 */
public static String writeImageDataToGcs(String googleId, byte[] imageData, String contentType) throws IOException {
    GcsFilename gcsFilename = new GcsFilename(Config.PRODUCTION_GCS_BUCKETNAME, googleId);
    try (GcsOutputChannel outputChannel =
            GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance())
            .createOrReplace(gcsFilename, new GcsFileOptions.Builder().mimeType(contentType).build())) {

        outputChannel.write(ByteBuffer.wrap(imageData));
    }

    return createBlobKey(googleId);
}
 
Example #6
Source File: GoogleIdMigrationBaseScript.java    From teammates with GNU General Public License v2.0 4 votes vote down vote up
@Override
protected void migrateEntity(Account oldAccount) throws Exception {
    String oldGoogleId = oldAccount.getGoogleId();
    String newGoogleId = generateNewGoogleId(oldAccount);

    Key<Account> oldAccountKey = Key.create(Account.class, oldAccount.getGoogleId());
    Key<StudentProfile> oldStudentProfileKey = Key.create(oldAccountKey, StudentProfile.class, oldGoogleId);
    StudentProfile oldStudentProfile = ofy().load().key(oldStudentProfileKey).now();

    List<CourseStudent> oldStudents = ofy().load().type(CourseStudent.class)
            .filter("googleId =", oldGoogleId).list();

    List<Instructor> oldInstructors = ofy().load().type(Instructor.class)
            .filter("googleId =", oldGoogleId).list();

    // update students and instructors

    if (!oldStudents.isEmpty()) {
        oldStudents.forEach(student -> student.setGoogleId(newGoogleId));
        ofy().save().entities(oldStudents).now();
    }

    if (!oldInstructors.isEmpty()) {
        oldInstructors.forEach(instructor -> instructor.setGoogleId(newGoogleId));
        ofy().save().entities(oldInstructors).now();
        instructorsDb.putDocuments(
                oldInstructors.stream().map(InstructorAttributes::valueOf).collect(Collectors.toList()));
    }

    // recreate account and student profile

    oldAccount.setGoogleId(newGoogleId);
    if (ofy().load().type(Account.class).id(newGoogleId).now() == null) {
        ofy().save().entity(oldAccount).now();
    } else {
        log(String.format("Skip creation of new account as account (%s) already exists", newGoogleId));
    }
    ofy().delete().type(Account.class).id(oldGoogleId).now();

    if (oldStudentProfile != null) {
        String pictureKey = oldStudentProfile.getPictureKey();

        if (!ClientProperties.isTargetUrlDevServer()) {
            try {
                GcsFilename oldGcsFilename = new GcsFilename(Config.PRODUCTION_GCS_BUCKETNAME, oldGoogleId);
                GcsFilename newGcsFilename = new GcsFilename(Config.PRODUCTION_GCS_BUCKETNAME, newGoogleId);
                GcsService gcsService = GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
                gcsService.copy(oldGcsFilename, newGcsFilename);
                gcsService.delete(oldGcsFilename);
                pictureKey = GoogleCloudStorageHelper.createBlobKey(newGoogleId);
            } catch (Exception e) {
                log("Profile picture not exist or error during copy: " + e.getMessage());
            }
        }

        oldStudentProfile.setGoogleId(newGoogleId);
        oldStudentProfile.setPictureKey(pictureKey);
        ofy().save().entity(oldStudentProfile).now();
        ofy().delete().key(oldStudentProfileKey).now();
    }

    log(String.format("Complete migration for account with googleId %s. The new googleId is %s",
            oldGoogleId, newGoogleId));
}