com.google.appengine.tools.cloudstorage.GcsServiceFactory Java Examples

/**
 * delete aia file based on given gallery id
 * @param galleryId gallery id
 */
private void deleteAIA(long galleryId) {
  try {
    //String galleryKey = GalleryApp.getSourceKey(galleryId);
    GallerySettings settings = loadGallerySettings();
    String galleryKey = settings.getSourceKey(galleryId);

    // setup cloud
    GcsService gcsService = GcsServiceFactory.createGcsService();
    //GcsFilename filename = new GcsFilename(GalleryApp.GALLERYBUCKET, galleryKey);
    GcsFilename filename = new GcsFilename(settings.getBucket(), galleryKey);
    gcsService.delete(filename);
  } catch (IOException e) {
    // TODO Auto-generated catch block
    LOG.log(Level.INFO, "FAILED GCS delete");
    e.printStackTrace();
  }
}
 

private void deleteImage(long galleryId) {
  try {
    //String galleryKey = GalleryApp.getImageKey(galleryId);
    GallerySettings settings = loadGallerySettings();
    String galleryKey = settings.getSourceKey(galleryId);
    // setup cloud
    GcsService gcsService = GcsServiceFactory.createGcsService();
    //GcsFilename filename = new GcsFilename(GalleryApp.GALLERYBUCKET, galleryKey);
    GcsFilename filename = new GcsFilename(settings.getBucket(), galleryKey);
    gcsService.delete(filename);
  } catch (IOException e) {
    // TODO Auto-generated catch block
    LOG.log(Level.INFO, "FAILED GCS delete");
    e.printStackTrace();
  }
}
 

ObjectifyStorageIo() {
  RetryParams retryParams = new RetryParams.Builder().initialRetryDelayMillis(100)
    .retryMaxAttempts(10)
    .totalRetryPeriodMillis(10000).build();
  if (DEBUG) {
    LOG.log(Level.INFO, "RetryParams: getInitialRetryDelayMillis() = " + retryParams.getInitialRetryDelayMillis());
    LOG.log(Level.INFO, "RetryParams: getRequestTimeoutMillis() = " + retryParams.getRequestTimeoutMillis());
    LOG.log(Level.INFO, "RetryParams: getRetryDelayBackoffFactor() = " + retryParams.getRetryDelayBackoffFactor());
    LOG.log(Level.INFO, "RetryParams: getRetryMaxAttempts() = " + retryParams.getRetryMaxAttempts());
    LOG.log(Level.INFO, "RetryParams: getRetryMinAttempts() = " + retryParams.getRetryMinAttempts());
    LOG.log(Level.INFO, "RetryParams: getTotalRetryPeriodMillis() = " + retryParams.getTotalRetryPeriodMillis());
  }
  gcsService = GcsServiceFactory.createGcsService(retryParams);
  memcache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO));
  initMotd();
}
 

@Before
public void before() throws Exception {
  // Force "development" mode so we don't try to really connect to GCS.
  SystemProperty.environment.set(SystemProperty.Environment.Value.Development);
  gcsService = GcsServiceFactory.createGcsService();

  createTld("tld");
  PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey();
  writeGcsFile(gcsService, GHOSTRYDE_FILE, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey));
  writeGcsFile(gcsService, GHOSTRYDE_R1_FILE, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey));
  writeGcsFile(gcsService, LENGTH_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8));
  writeGcsFile(gcsService, LENGTH_R1_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8));
  writeGcsFile(gcsService, REPORT_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey));
  writeGcsFile(gcsService, REPORT_R1_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey));
  tm()
      .transact(
          () -> {
            RdeRevision.saveRevision("lol", DateTime.parse("2010-10-17TZ"), FULL, 0);
            RdeRevision.saveRevision("tld", DateTime.parse("2010-10-17TZ"), FULL, 0);
          });
}
 

@Test
@InSequence(100)
public void testCreateGsBlobKey() throws Exception {
    final long ts = System.currentTimeMillis();
    final byte[] bytes = "FooBar".getBytes();

    GcsService service = GcsServiceFactory.createGcsService();
    GcsFilename filename = new GcsFilename("GcsBucket", String.valueOf(ts));
    GcsFileOptions options = new GcsFileOptions.Builder().mimeType(CONTENT_TYPE).build();
    try (GcsOutputChannel out = service.createOrReplace(filename, options)) {
        IOUtils.copy(Channels.newChannel(new ByteArrayInputStream(bytes)), out);
    }

    BlobstoreService blobstoreService = BlobstoreServiceFactory.getBlobstoreService();
    BlobKey key =  blobstoreService.createGsBlobKey("/gs/GcsBucket/" + ts);
    byte[] fetched = blobstoreService.fetchData(key, 0, 10);
    Assert.assertArrayEquals(bytes, fetched);
}
 

/**
 * store aia file on cloud server
 * @param galleryId gallery id
 * @param projectId project id
 * @param projectName project name
 */
private void storeAIA(long galleryId, long projectId, String projectName) throws IOException {

  final String userId = userInfoProvider.getUserId();
  // build the aia file name using the ai project name and code stolen
  // from DownloadServlet to normalize...
  String aiaName = StringUtils.normalizeForFilename(projectName) + ".aia";
  // grab the data for the aia file using code from DownloadServlet
  RawFile aiaFile = null;
  byte[] aiaBytes= null;
  ProjectSourceZip zipFile = fileExporter.exportProjectSourceZip(userId,
    projectId, true, false, aiaName, false, false, false, true);
  aiaFile = zipFile.getRawFile();
  aiaBytes = aiaFile.getContent();
  LOG.log(Level.INFO, "aiaFile numBytes:"+aiaBytes.length);
  // now stick the aia file into the gcs

  //String galleryKey = GalleryApp.getSourceKey(galleryId);//String.valueOf(galleryId);
  GallerySettings settings = loadGallerySettings();
  String galleryKey = settings.getSourceKey(galleryId);
  // setup cloud
  GcsService gcsService = GcsServiceFactory.createGcsService();

  //GcsFilename filename = new GcsFilename(GalleryApp.GALLERYBUCKET, galleryKey);
  GcsFilename filename = new GcsFilename(settings.getBucket(), galleryKey);

  GcsFileOptions options = new GcsFileOptions.Builder().mimeType("application/zip")
    .acl("public-read").cacheControl("no-cache").addUserMetadata("title", aiaName).build();
  GcsOutputChannel writeChannel = gcsService.createOrReplace(filename, options);
  writeChannel.write(ByteBuffer.wrap(aiaBytes));

  // Now finalize
  writeChannel.close();
}
 

/**
 * Writes a byte array {@code imageData} as image to the Google Cloud Storage,
 * with the {@code googleId} as the identifier name for the image.
 *
 * @return the {@link BlobKey} used as the image's identifier in Google Cloud Storage
 */
public static String writeImageDataToGcs(String googleId, byte[] imageData, String contentType) throws IOException {
    GcsFilename gcsFilename = new GcsFilename(Config.PRODUCTION_GCS_BUCKETNAME, googleId);
    try (GcsOutputChannel outputChannel =
            GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance())
            .createOrReplace(gcsFilename, new GcsFileOptions.Builder().mimeType(contentType).build())) {

        outputChannel.write(ByteBuffer.wrap(imageData));
    }

    return createBlobKey(googleId);
}
 

@Before
public void setUp() {
    bucket = getTestSystemProperty("tck.gcs.bucket");
    if (bucket == null) {
        bucket = SystemProperty.applicationId.get() + ".appspot.com";
    }
    gcsService = GcsServiceFactory.createGcsService();
}
 

/**
 * when an app is published/updated, we need to move the image
 * that was temporarily uploaded into projects/projectid/image
 * into the gallery image
 * @param app gallery app
 */
private void setGalleryAppImage(GalleryApp app) {
  // best thing would be if GCS has a mv op, we can just do that.
  // don't think that is there, though, so for now read one and write to other
  // First, read the file from projects name
  boolean lockForRead = false;
  //String projectImageKey = app.getProjectImageKey();
  GallerySettings settings = loadGallerySettings();
  String projectImageKey = settings.getProjectImageKey(app.getProjectId());
  try {
    GcsService gcsService = GcsServiceFactory.createGcsService();
    //GcsFilename filename = new GcsFilename(GalleryApp.GALLERYBUCKET, projectImageKey);
    GcsFilename filename = new GcsFilename(settings.getBucket(), projectImageKey);
    GcsInputChannel readChannel = gcsService.openReadChannel(filename, 0);
    InputStream gcsis = Channels.newInputStream(readChannel);

    byte[] buffer = new byte[8000];
    int bytesRead = 0;
    ByteArrayOutputStream bao = new ByteArrayOutputStream();

    while ((bytesRead = gcsis.read(buffer)) != -1) {
      bao.write(buffer, 0, bytesRead);
    }
    // close the project image file
    readChannel.close();

    // if image is greater than 200 X 200, it will be scaled (200 X 200).
    // otherwise, it will be stored as origin.
    byte[] oldImageData = bao.toByteArray();
    byte[] newImageData;
    ImagesService imagesService = ImagesServiceFactory.getImagesService();
    Image oldImage = ImagesServiceFactory.makeImage(oldImageData);
    //if image size is too big, scale it to a smaller size.
    if(oldImage.getWidth() > 200 && oldImage.getHeight() > 200){
        Transform resize = ImagesServiceFactory.makeResize(200, 200);
        Image newImage = imagesService.applyTransform(resize, oldImage);
        newImageData = newImage.getImageData();
    }else{
        newImageData = oldImageData;
    }

    // set up the cloud file (options)
    // After publish, copy the /projects/projectId image into /apps/appId
    //String galleryKey = app.getImageKey();
    String galleryKey = settings.getImageKey(app.getGalleryAppId());

    //GcsFilename outfilename = new GcsFilename(GalleryApp.GALLERYBUCKET, galleryKey);
    GcsFilename outfilename = new GcsFilename(settings.getBucket(), galleryKey);
    GcsFileOptions options = new GcsFileOptions.Builder().mimeType("image/jpeg")
        .acl("public-read").cacheControl("no-cache").build();
    GcsOutputChannel writeChannel = gcsService.createOrReplace(outfilename, options);
    writeChannel.write(ByteBuffer.wrap(newImageData));

    // Now finalize
    writeChannel.close();

  } catch (IOException e) {
    // TODO Auto-generated catch block
    LOG.log(Level.INFO, "FAILED WRITING IMAGE TO GCS");
    e.printStackTrace();
  }
}
 

@Override
  public void doPost(HttpServletRequest req, HttpServletResponse resp) {
    setDefaultHeader(resp);
    UploadResponse uploadResponse;

    String uri = req.getRequestURI();
    // First, call split with no limit parameter.
    String[] uriComponents = uri.split("/");

    if (true) {
      String requestType = uriComponents[REQUEST_TYPE_INDEX];
      if (DEBUG) {
        LOG.info("######### GOT IN URI");
        LOG.info(requestType);
      }

      long project_Id = -1;
      String user_Id = "-1";
      if (requestType.equalsIgnoreCase("apps")) {
        project_Id = Long.parseLong(uriComponents[GALLERY_OR_USER_ID_INDEX]);
      } else if (requestType.equalsIgnoreCase("user")) {
        //the code below doesn't check if user_Id is the id of current user
        //user_Id = uriComponents[GALLERY_OR_USER_ID_INDEX];
        user_Id = userInfoProvider.getUserId();
      }
      InputStream uploadedStream;
      try {
        if(req.getContentLength() < MAX_IMAGE_FILE_SIZE){
          uploadedStream = getRequestStream(req, ServerLayout.UPLOAD_FILE_FORM_ELEMENT);

          // Converts the input stream to byte array
          byte[] buffer = new byte[8000];
          int bytesRead = 0;
          ByteArrayOutputStream bao = new ByteArrayOutputStream();
          while ((bytesRead = uploadedStream.read(buffer)) != -1) {
            bao.write(buffer, 0, bytesRead);
          }
          // Set up the cloud file (options)
          String key = "";
          GallerySettings settings = galleryService.loadGallerySettings();
          if (requestType.equalsIgnoreCase("apps")) {
            key = settings.getProjectImageKey(project_Id);
          } else if (requestType.equalsIgnoreCase("user")) {
            key =  settings.getUserImageKey(user_Id);
          }

          // setup cloud
          GcsService gcsService = GcsServiceFactory.createGcsService();
          GcsFilename filename = new GcsFilename(settings.getBucket(), key);
          GcsFileOptions options = new GcsFileOptions.Builder().mimeType("image/jpeg")
                  .acl("public-read").cacheControl("no-cache").build();
          GcsOutputChannel writeChannel = gcsService.createOrReplace(filename, options);
          writeChannel.write(ByteBuffer.wrap(bao.toByteArray()));

          // Now finalize
          writeChannel.close();

          uploadResponse = new UploadResponse(UploadResponse.Status.SUCCESS);
        }else{
          /*file exceeds size of MAX_IMAGE_FILE_SIZE*/
          uploadResponse = new UploadResponse(UploadResponse.Status.FILE_TOO_LARGE);
        }

        // Now, get the PrintWriter for the servlet response and print the UploadResponse.
        // On the client side, in the onSubmitComplete method in ode/client/utils/Uploader.java, the
        // UploadResponse value will be retrieved as a String via the
        // FormSubmitCompleteEvent.getResults() method.
        PrintWriter out = resp.getWriter();
        out.print(uploadResponse.formatAsHtml());

      } catch (Exception e) {
        throw CrashReport.createAndLogError(LOG, req, null, e);
      }
      // Set http response information
      resp.setStatus(HttpServletResponse.SC_OK);
    }

    // Now, get the PrintWriter for the servlet response and print the UploadResponse.
    // On the client side, in the onSubmitComplete method in ode/client/utils/Uploader.java, the
    // UploadResponse value will be retrieved as a String via the
    // FormSubmitCompleteEvent.getResults() method.
//    PrintWriter out = resp.getWriter();
//    out.print(uploadResponse.formatAsHtml());

    // Set http response information
    resp.setStatus(HttpServletResponse.SC_OK);
  }
 

@Override
protected void migrateEntity(Account oldAccount) throws Exception {
    String oldGoogleId = oldAccount.getGoogleId();
    String newGoogleId = generateNewGoogleId(oldAccount);

    Key<Account> oldAccountKey = Key.create(Account.class, oldAccount.getGoogleId());
    Key<StudentProfile> oldStudentProfileKey = Key.create(oldAccountKey, StudentProfile.class, oldGoogleId);
    StudentProfile oldStudentProfile = ofy().load().key(oldStudentProfileKey).now();

    List<CourseStudent> oldStudents = ofy().load().type(CourseStudent.class)
            .filter("googleId =", oldGoogleId).list();

    List<Instructor> oldInstructors = ofy().load().type(Instructor.class)
            .filter("googleId =", oldGoogleId).list();

    // update students and instructors

    if (!oldStudents.isEmpty()) {
        oldStudents.forEach(student -> student.setGoogleId(newGoogleId));
        ofy().save().entities(oldStudents).now();
    }

    if (!oldInstructors.isEmpty()) {
        oldInstructors.forEach(instructor -> instructor.setGoogleId(newGoogleId));
        ofy().save().entities(oldInstructors).now();
        instructorsDb.putDocuments(
                oldInstructors.stream().map(InstructorAttributes::valueOf).collect(Collectors.toList()));
    }

    // recreate account and student profile

    oldAccount.setGoogleId(newGoogleId);
    if (ofy().load().type(Account.class).id(newGoogleId).now() == null) {
        ofy().save().entity(oldAccount).now();
    } else {
        log(String.format("Skip creation of new account as account (%s) already exists", newGoogleId));
    }
    ofy().delete().type(Account.class).id(oldGoogleId).now();

    if (oldStudentProfile != null) {
        String pictureKey = oldStudentProfile.getPictureKey();

        if (!ClientProperties.isTargetUrlDevServer()) {
            try {
                GcsFilename oldGcsFilename = new GcsFilename(Config.PRODUCTION_GCS_BUCKETNAME, oldGoogleId);
                GcsFilename newGcsFilename = new GcsFilename(Config.PRODUCTION_GCS_BUCKETNAME, newGoogleId);
                GcsService gcsService = GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
                gcsService.copy(oldGcsFilename, newGcsFilename);
                gcsService.delete(oldGcsFilename);
                pictureKey = GoogleCloudStorageHelper.createBlobKey(newGoogleId);
            } catch (Exception e) {
                log("Profile picture not exist or error during copy: " + e.getMessage());
            }
        }

        oldStudentProfile.setGoogleId(newGoogleId);
        oldStudentProfile.setPictureKey(pictureKey);
        ofy().save().entity(oldStudentProfile).now();
        ofy().delete().key(oldStudentProfileKey).now();
    }

    log(String.format("Complete migration for account with googleId %s. The new googleId is %s",
            oldGoogleId, newGoogleId));
}