Java Code Examples for org.apache.beam.sdk.io.FileSystems#create()

public static String copyFile(ResourceId sourceFile, ResourceId destinationFile)
    throws IOException {

  try (WritableByteChannel writeChannel = FileSystems.create(destinationFile, "text/plain")) {
    try (ReadableByteChannel readChannel = FileSystems.open(sourceFile)) {

      final ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024);
      while (readChannel.read(buffer) != -1) {
        buffer.flip();
        writeChannel.write(buffer);
        buffer.compact();
      }
      buffer.flip();
      while (buffer.hasRemaining()) {
        writeChannel.write(buffer);
      }
    }
  }

  return destinationFile.toString();
}
 

public static String copyFile(ResourceId sourceFile, ResourceId destinationFile)
    throws IOException {

  try (WritableByteChannel writeChannel = FileSystems.create(destinationFile, "text/plain")) {
    try (ReadableByteChannel readChannel = FileSystems.open(sourceFile)) {

      final ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024);
      while (readChannel.read(buffer) != -1) {
        buffer.flip();
        writeChannel.write(buffer);
        buffer.compact();
      }
      buffer.flip();
      while (buffer.hasRemaining()) {
        writeChannel.write(buffer);
      }
    }
  }

  return destinationFile.toString();
}
 

private void writeRowsHelper(
    List<TableRow> rows, Schema avroSchema, String destinationPattern, int shard) {
  String filename = destinationPattern.replace("*", String.format("%012d", shard));
  try (WritableByteChannel channel =
          FileSystems.create(
              FileSystems.matchNewResource(filename, false /* isDirectory */), MimeTypes.BINARY);
      DataFileWriter<GenericRecord> tableRowWriter =
          new DataFileWriter<>(new GenericDatumWriter<GenericRecord>(avroSchema))
              .create(avroSchema, Channels.newOutputStream(channel))) {
    for (Map<String, Object> record : rows) {
      GenericRecordBuilder genericRecordBuilder = new GenericRecordBuilder(avroSchema);
      for (Map.Entry<String, Object> field : record.entrySet()) {
        genericRecordBuilder.set(field.getKey(), field.getValue());
      }
      tableRowWriter.append(genericRecordBuilder.build());
    }
  } catch (IOException e) {
    throw new IllegalStateException(
        String.format("Could not create destination for extract job %s", filename), e);
  }
}
 

private StagingResult tryStagePackage(PackageAttributes attributes, CreateOptions createOptions)
    throws IOException, InterruptedException {
  String sourceDescription = attributes.getSourceDescription();
  String target = attributes.getDestination().getLocation();

  LOG.info("Uploading {} to {}", sourceDescription, target);
  try (WritableByteChannel writer =
      FileSystems.create(FileSystems.matchNewResource(target, false), createOptions)) {
    if (attributes.getBytes() != null) {
      ByteSource.wrap(attributes.getBytes()).copyTo(Channels.newOutputStream(writer));
    } else {
      File sourceFile = attributes.getSource();
      checkState(
          sourceFile != null,
          "Internal inconsistency: we tried to stage something to %s, but neither a source file "
              + "nor the byte content was specified",
          target);
      Files.asByteSource(sourceFile).copyTo(Channels.newOutputStream(writer));
    }
  }
  return StagingResult.uploaded(attributes);
}
 

/**
* Write to cloud storage using the FileSystems API. See https://stackoverflow.com/a/50050583.
*/
private void writeToStorage(String path, byte[] data) throws Exception {
  ResourceId resourceId = FileSystems.matchNewResource(path, false);
  try (ByteArrayInputStream inputStream = new ByteArrayInputStream(data);
      ReadableByteChannel readerChannel = Channels.newChannel(inputStream);
      WritableByteChannel writerChannel = FileSystems.create(resourceId, MimeTypes.BINARY)) {
    ByteStreams.copy(readerChannel, writerChannel);
  }
}
 

/**
 * Helper to generate files for testing.
 *
 * @param filePath The path to the file to write.
 * @param fileContents The content to write.
 * @return The file written.
 * @throws IOException If an error occurs while creating or writing the file.
 */
private static ResourceId writeToFile(String filePath, String fileContents) throws IOException {

  ResourceId resourceId = FileSystems.matchNewResource(filePath, false);

  // Write the file contents to the channel and close.
  try (ReadableByteChannel readChannel =
      Channels.newChannel(new ByteArrayInputStream(fileContents.getBytes()))) {
    try (WritableByteChannel writeChannel = FileSystems.create(resourceId, MimeTypes.TEXT)) {
      ByteStreams.copy(readChannel, writeChannel);
    }
  }

  return resourceId;
}
 

/**
 * Decompresses the inputFile using the specified compression and outputs to the main output of
 * the {@link Decompress} doFn. Files output to the destination will be first written as temp
 * files with a "temp-" prefix within the output directory. If a file fails decompression, the
 * filename and the associated error will be output to the dead-letter.
 *
 * @param inputFile The inputFile to decompress.
 * @return A {@link ResourceId} which points to the resulting file from the decompression.
 */
private ResourceId decompress(ResourceId inputFile) throws IOException {
  // Remove the compressed extension from the file. Example: demo.txt.gz -> demo.txt
  String outputFilename = Files.getNameWithoutExtension(inputFile.toString());

  // Resolve the necessary resources to perform the transfer.
  ResourceId outputDir = FileSystems.matchNewResource(destinationLocation.get(), true);
  ResourceId outputFile =
      outputDir.resolve(outputFilename, StandardResolveOptions.RESOLVE_FILE);
  ResourceId tempFile =
      outputDir.resolve(Files.getFileExtension(inputFile.toString())
          + "-temp-" + outputFilename, StandardResolveOptions.RESOLVE_FILE);

  // Resolve the compression
  Compression compression = Compression.detect(inputFile.toString());

  // Perform the copy of the decompressed channel into the destination.
  try (ReadableByteChannel readerChannel =
      compression.readDecompressed(FileSystems.open(inputFile))) {
    try (WritableByteChannel writerChannel = FileSystems.create(tempFile, MimeTypes.TEXT)) {
      ByteStreams.copy(readerChannel, writerChannel);
    }

    // Rename the temp file to the output file.
    FileSystems.rename(
        ImmutableList.of(tempFile),
        ImmutableList.of(outputFile),
        MoveOptions.StandardMoveOptions.IGNORE_MISSING_FILES);
  } catch (IOException e) {
    String msg = e.getMessage();

    LOG.error("Error occurred during decompression of {}", inputFile.toString(), e);
    throw new IOException(sanitizeDecompressionErrorMsg(msg, inputFile, compression));
  }

  return outputFile;
}
 

private void saveSchema(String content, String schemaPath) {
  LOG.info("Schema: " + content);

  try {
    WritableByteChannel chan =
        FileSystems.create(FileSystems.matchNewResource(schemaPath, false), "text/plain");
    try (OutputStream stream = Channels.newOutputStream(chan)) {
      stream.write(content.getBytes());
    }
  } catch (IOException e) {
    throw new RuntimeException("Failed to write schema", e);
  }
}
 

/** utility method to copy binary (jar file) data from source to dest. */
private static void copy(ResourceId source, ResourceId dest) throws IOException {
  try (ReadableByteChannel rbc = FileSystems.open(source)) {
    try (WritableByteChannel wbc = FileSystems.create(dest, MimeTypes.BINARY)) {
      ByteStreams.copy(rbc, wbc);
    }
  }
}
 

public static void writeToFile(final String filename, final ByteBuffer contents)
    throws IOException {
  ResourceId resourceId = FileSystems.matchNewResource(filename, false);
  try (WritableByteChannel out = FileSystems.create(resourceId, MimeTypes.TEXT)) {
    out.write(contents);
  }
}
 

BigQueryRowWriter(String basename, String mimeType) throws Exception {
  String uId = UUID.randomUUID().toString();
  resourceId = FileSystems.matchNewResource(basename + uId, false);
  LOG.info("Opening {} to {}.", this.getClass().getSimpleName(), resourceId);
  channel = FileSystems.create(resourceId, mimeType);
  out = new CountingOutputStream(Channels.newOutputStream(channel));
}
 

/**
 * Init batch.
 *
 * @throws IOException the io exception
 */
@StartBundle
public void initFile() throws IOException {
  // Write each bundle to newline delimited JSON file.
  String filename = String.format("fhirImportBatch-%s.ndjson", UUID.randomUUID().toString());
  ResourceId tempDir = FileSystems.matchNewResource(this.tempGcsPath.get(), true);
  this.resourceId = tempDir.resolve(filename, StandardResolveOptions.RESOLVE_FILE);
  this.ndJsonChannel = FileSystems.create(resourceId, "application/ld+json");
  if (mapper == null) {
    this.mapper = new ObjectMapper();
  }
}
 

private void uploadFileToGCS(File srcPath, ResourceId destination) throws IOException {
  StandardCreateOptions createOptions =
      StandardCreateOptions.builder().setMimeType("application/octet-stream").build();
  try (WritableByteChannel dst = FileSystems.create(destination, createOptions)) {
    try (ReadableByteChannel src = Channels.newChannel(new FileInputStream(srcPath))) {
      ByteStreams.copy(src, dst);
    }
  }
}
 

public static void writeTextToFileSideEffect(String text, String filename) throws IOException {
  ResourceId rid = FileSystems.matchNewResource(filename, false);
  WritableByteChannel chan = FileSystems.create(rid, "text/plain");
  chan.write(ByteBuffer.wrap(text.getBytes(StandardCharsets.UTF_8)));
  chan.close();
}
 

@Override
public SinkWriter<WindowedValue<IsmRecord<V>>> writer() throws IOException {
  return new IsmSinkWriter(FileSystems.create(resourceId, MimeTypes.BINARY));
}
 

public AvroByteFileWriter() throws IOException {
  WritableByteChannel writer = FileSystems.create(resourceId, MimeTypes.BINARY);

  fileWriter = new DataFileWriter<>(new GenericDatumWriter<ByteBuffer>(schema));
  fileWriter.create(schema, Channels.newOutputStream(writer));
}