Java Code Examples for org.apache.beam.sdk.values.PDone

The following examples show how to use org.apache.beam.sdk.values.PDone. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: gcp-ingestion   Source File: RepublishPerDocType.java    License: Mozilla Public License 2.0 7 votes vote down vote up
@Override
public PDone expand(PCollection<PubsubMessage> input) {
  List<Destination> destinations = baseOptions.getPerDocTypeDestinations().entrySet().stream()
      .flatMap(
          entry -> entry.getValue().stream().map(value -> new Destination(entry.getKey(), value)))
      .collect(Collectors.toList());

  int numDestinations = destinations.size();
  int numPartitions = numDestinations + 1;
  PCollectionList<PubsubMessage> partitioned = input.apply("PartitionByDocType",
      Partition.of(numPartitions, new PartitionFn(destinations)));

  for (int i = 0; i < numDestinations; i++) {
    Destination destination = destinations.get(i);
    RepublisherOptions.Parsed opts = baseOptions.as(RepublisherOptions.Parsed.class);
    opts.setOutput(StaticValueProvider.of(destination.dest));
    String name = String.join("_", "republish", destination.namespace, destination.docType);
    partitioned.get(i).apply(name, opts.getOutputType().write(opts));
  }

  return PDone.in(input.getPipeline());
}
 
Example 2
@Override
public PDone expand(PCollection<PubsubMessage> input) {
  List<Destination> destinations = baseOptions.getPerNamespaceDestinations().entrySet().stream()
      .map(entry -> new Destination(entry.getKey(), entry.getValue()))
      .collect(Collectors.toList());
  int numDestinations = destinations.size();
  int numPartitions = numDestinations + 1;
  PCollectionList<PubsubMessage> partitioned = input.apply("PartitionByNamespace",
      Partition.of(numPartitions, new PartitionFn(destinations)));

  for (int i = 0; i < numDestinations; i++) {
    Destination destination = destinations.get(i);
    RepublisherOptions.Parsed opts = baseOptions.as(RepublisherOptions.Parsed.class);
    opts.setOutput(StaticValueProvider.of(destination.dest));
    String name = String.join("_", "republish", destination.namespace);
    partitioned.get(i).apply(name, opts.getOutputType().write(opts));
  }

  return PDone.in(input.getPipeline());
}
 
Example 3
Source Project: beam   Source File: JdbcIO.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public PDone expand(PCollection<T> input) {
  // fixme: validate invalid table input
  if (input.hasSchema() && !hasStatementAndSetter()) {
    checkArgument(
        inner.getTable() != null, "table cannot be null if statement is not provided");
    Schema schema = input.getSchema();
    List<SchemaUtil.FieldWithIndex> fields = getFilteredFields(schema);
    inner =
        inner.withStatement(
            JdbcUtil.generateStatement(
                inner.getTable(),
                fields.stream()
                    .map(SchemaUtil.FieldWithIndex::getField)
                    .collect(Collectors.toList())));
    inner =
        inner.withPreparedStatementSetter(
            new AutoGeneratedPreparedStatementSetter(fields, input.getToRowFunction()));
  }

  inner.expand(input);
  return PDone.in(input.getPipeline());
}
 
Example 4
Source Project: feast   Source File: WriteFailureMetricsTransform.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public PDone expand(PCollection<FailedElement> input) {
  ImportOptions options = input.getPipeline().getOptions().as(ImportOptions.class);
  if ("statsd".equals(options.getMetricsExporterType())) {
    input.apply(
        "WriteDeadletterMetrics",
        ParDo.of(
            WriteDeadletterRowMetricsDoFn.newBuilder()
                .setStatsdHost(options.getStatsdHost())
                .setStatsdPort(options.getStatsdPort())
                .setStoreName(getStoreName())
                .build()));
  } else {
    input.apply(
        "Noop",
        ParDo.of(
            new DoFn<FailedElement, Void>() {
              @ProcessElement
              public void processElement(ProcessContext c) {}
            }));
  }
  return PDone.in(input.getPipeline());
}
 
Example 5
Source Project: beam   Source File: XmlIO.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public PDone expand(PCollection<T> input) {
  checkArgument(getRecordClass() != null, "withRecordClass() is required");
  checkArgument(getRootElement() != null, "withRootElement() is required");
  checkArgument(getFilenamePrefix() != null, "to() is required");
  checkArgument(getCharset() != null, "withCharset() is required");
  try {
    JAXBContext.newInstance(getRecordClass());
  } catch (JAXBException e) {
    throw new RuntimeException("Error binding classes to a JAXB Context.", e);
  }

  ResourceId prefix =
      FileSystems.matchNewResource(getFilenamePrefix(), false /* isDirectory */);
  input.apply(
      FileIO.<T>write()
          .via(
              sink(getRecordClass())
                  .withCharset(Charset.forName(getCharset()))
                  .withRootElement(getRootElement()))
          .to(prefix.getCurrentDirectory().toString())
          .withPrefix(prefix.getFilename())
          .withSuffix(".xml")
          .withIgnoreWindowing());
  return PDone.in(input.getPipeline());
}
 
Example 6
Source Project: feast   Source File: BigQueryDeadletterSink.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public PDone expand(PCollection<FailedElement> input) {
  TimePartitioning partition = new TimePartitioning().setType("DAY");
  partition.setField(TIMESTAMP_COLUMN);
  input
      .apply("FailedElementToTableRow", ParDo.of(new FailedElementToTableRowFn()))
      .apply(
          "WriteFailedElementsToBigQuery",
          BigQueryIO.writeTableRows()
              .to(getTableSpec())
              .withJsonSchema(getJsonSchema())
              .withTimePartitioning(partition)
              .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
              .withWriteDisposition(WriteDisposition.WRITE_APPEND));
  return PDone.in(input.getPipeline());
}
 
Example 7
Source Project: beam   Source File: PTransformTranslationTest.java    License: Apache License 2.0 6 votes vote down vote up
private static AppliedPTransform<?, ?, ?> rawPTransformWithNullSpec(Pipeline pipeline) {
  PTransformTranslation.RawPTransform<PBegin, PDone> rawPTransform =
      new PTransformTranslation.RawPTransform<PBegin, PDone>() {
        @Override
        public String getUrn() {
          return "fake/urn";
        }

        @Nullable
        @Override
        public RunnerApi.FunctionSpec getSpec() {
          return null;
        }
      };
  return AppliedPTransform.<PBegin, PDone, PTransform<PBegin, PDone>>of(
      "RawPTransformWithNoSpec",
      pipeline.begin().expand(),
      PDone.in(pipeline).expand(),
      rawPTransform,
      pipeline);
}
 
Example 8
Source Project: beam   Source File: ClickHouseIO.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public PDone expand(PCollection<T> input) {
  TableSchema tableSchema = getTableSchema(jdbcUrl(), table());
  Properties properties = properties();

  set(properties, ClickHouseQueryParam.MAX_INSERT_BLOCK_SIZE, maxInsertBlockSize());
  set(properties, ClickHouseQueryParam.INSERT_QUORUM, insertQuorum());
  set(properties, "insert_distributed_sync", insertDistributedSync());
  set(properties, "insert_deduplication", insertDeduplicate());

  WriteFn<T> fn =
      new AutoValue_ClickHouseIO_WriteFn.Builder<T>()
          .jdbcUrl(jdbcUrl())
          .table(table())
          .maxInsertBlockSize(maxInsertBlockSize())
          .schema(tableSchema)
          .properties(properties)
          .initialBackoff(initialBackoff())
          .maxCumulativeBackoff(maxCumulativeBackoff())
          .maxRetries(maxRetries())
          .build();

  input.apply(ParDo.of(fn));

  return PDone.in(input.getPipeline());
}
 
Example 9
Source Project: beam   Source File: TfIdf.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public PDone expand(PCollection<KV<String, KV<URI, Double>>> wordToUriAndTfIdf) {
  return wordToUriAndTfIdf
      .apply(
          "Format",
          ParDo.of(
              new DoFn<KV<String, KV<URI, Double>>, String>() {
                @ProcessElement
                public void processElement(ProcessContext c) {
                  c.output(
                      String.format(
                          "%s,\t%s,\t%f",
                          c.element().getKey(),
                          c.element().getValue().getKey(),
                          c.element().getValue().getValue()));
                }
              }))
      .apply(TextIO.write().to(output).withSuffix(".csv"));
}
 
Example 10
Source Project: beam   Source File: ExpansionService.java    License: Apache License 2.0 6 votes vote down vote up
default Map<String, PCollection<?>> extractOutputs(OutputT output) {
  if (output instanceof PDone) {
    return Collections.emptyMap();
  } else if (output instanceof PCollection) {
    return ImmutableMap.of("output", (PCollection<?>) output);
  } else if (output instanceof PCollectionTuple) {
    return ((PCollectionTuple) output)
        .getAll().entrySet().stream()
            .collect(Collectors.toMap(entry -> entry.getKey().getId(), Map.Entry::getValue));
  } else if (output instanceof PCollectionList<?>) {
    PCollectionList<?> listOutput = (PCollectionList<?>) output;
    return IntStream.range(0, listOutput.size())
        .boxed()
        .collect(Collectors.toMap(Object::toString, listOutput::get));
  } else {
    throw new UnsupportedOperationException("Unknown output type: " + output.getClass());
  }
}
 
Example 11
Source Project: beam   Source File: TestBoundedTable.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public POutput buildIOWriter(PCollection<Row> input) {
  input.apply(
      ParDo.of(
          new DoFn<Row, Void>() {
            @ProcessElement
            public void processElement(ProcessContext c) {
              CONTENT.add(c.element());
            }

            @Teardown
            public void close() {
              CONTENT.clear();
            }
          }));
  return PDone.in(input.getPipeline());
}
 
Example 12
@Override
public PDone expand(PBegin begin) {
  PCollection<Boolean> result =
      begin
          .apply(Create.of(DUMMY_ROW).withRowSchema(DUMMY_SCHEMA))
          .apply(SqlTransform.query("SELECT " + expr))
          .apply(MapElements.into(TypeDescriptors.booleans()).via(row -> row.getBoolean(0)));

  PAssert.that(result)
      .satisfies(
          input -> {
            assertTrue("Test expression is false: " + expr, Iterables.getOnlyElement(input));
            return null;
          });
  return PDone.in(begin.getPipeline());
}
 
Example 13
Source Project: beam   Source File: KafkaIO.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public PDone expand(PCollection<KV<K, V>> input) {
  checkArgument(getTopic() != null, "withTopic() is required");

  KvCoder<K, V> kvCoder = (KvCoder<K, V>) input.getCoder();
  return input
      .apply(
          "Kafka ProducerRecord",
          MapElements.via(
              new SimpleFunction<KV<K, V>, ProducerRecord<K, V>>() {
                @Override
                public ProducerRecord<K, V> apply(KV<K, V> element) {
                  return new ProducerRecord<>(getTopic(), element.getKey(), element.getValue());
                }
              }))
      .setCoder(ProducerRecordCoder.of(kvCoder.getKeyCoder(), kvCoder.getValueCoder()))
      .apply(getWriteRecordsTransform());
}
 
Example 14
Source Project: beam   Source File: NameUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testPTransformNameWithAnonOuterClass() throws Exception {
  AnonymousClass anonymousClassObj =
      new AnonymousClass() {
        class NamedInnerClass extends PTransform<PBegin, PDone> {
          @Override
          public PDone expand(PBegin begin) {
            throw new IllegalArgumentException("Should never be applied");
          }
        }

        @Override
        public Object getInnerClassInstance() {
          return new NamedInnerClass();
        }
      };

  assertEquals(
      "NamedInnerClass",
      NameUtils.approximateSimpleName(anonymousClassObj.getInnerClassInstance()));
  assertEquals(
      "NameUtilsTest.NamedInnerClass",
      NameUtils.approximatePTransformName(anonymousClassObj.getInnerClassInstance().getClass()));
}
 
Example 15
Source Project: beam   Source File: DirectRunnerTest.java    License: Apache License 2.0 6 votes vote down vote up
private PTransform<PBegin, PDone> outputStartTo(StaticQueue<Integer> queue) {
  return new PTransform<PBegin, PDone>() {
    @Override
    public PDone expand(PBegin input) {
      input
          .apply(Create.of(1))
          .apply(
              MapElements.into(TypeDescriptors.voids())
                  .via(
                      in -> {
                        queue.add(in);
                        return null;
                      }));
      return PDone.in(input.getPipeline());
    }
  };
}
 
Example 16
Source Project: beam   Source File: DatastoreV1.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public PDone expand(PCollection<T> input) {
  checkArgument(projectId != null, "withProjectId() is required");
  if (projectId.isAccessible()) {
    checkArgument(projectId.get() != null, "projectId can not be null");
  }
  checkArgument(mutationFn != null, "mutationFn can not be null");

  input
      .apply("Convert to Mutation", MapElements.via(mutationFn))
      .apply(
          "Write Mutation to Datastore", ParDo.of(new DatastoreWriterFn(projectId, localhost)));

  return PDone.in(input.getPipeline());
}
 
Example 17
@Override
public PDone expand(PCollection<String> input) {

  ResourceId resource = FileBasedSink.convertToFileResourceIfPossible(filenamePrefix);
  TextIO.Write write =
      TextIO.write()
          .to(new PerWindowFiles(resource))
          .withTempDirectory(resource.getCurrentDirectory())
          .withWindowedWrites();

  if (numShards != null) {
    write = write.withNumShards(numShards);
  }

  return input.apply(write);
}
 
Example 18
Source Project: beam   Source File: SnowflakeIO.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public PDone expand(PCollection<T> input) {
  checkArguments();

  String stagingBucketDir = String.format("%s/%s/", getStagingBucketName(), WRITE_TMP_PATH);

  PCollection<String> out = write(input, stagingBucketDir);
  out.setCoder(StringUtf8Coder.of());

  return PDone.in(out.getPipeline());
}
 
Example 19
Source Project: incubator-nemo   Source File: WriteOneFilePerWindow.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public PDone expand(final PCollection<String> input) {
  final ResourceId resource = FileBasedSink.convertToFileResourceIfPossible(filenamePrefix);
  TextIO.Write write =
    TextIO.write()
      .to(new PerWindowFiles(resource))
      .withTempDirectory(resource.getCurrentDirectory())
      .withWindowedWrites();
  if (numShards != null) {
    write = write.withNumShards(numShards);
  }
  return input.apply(write);
}
 
Example 20
Source Project: incubator-nemo   Source File: GenericSourceSink.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Write data.
 * NEMO-365: This method could later be replaced using the HadoopFormatIO class.
 *
 * @param dataToWrite data to write
 * @param path        path to write data
 * @return returns {@link PDone}
 */
public static PDone write(final PCollection<String> dataToWrite,
                          final String path) {
  if (isHDFSPath(path)) {
    dataToWrite.apply(ParDo.of(new HDFSWrite(path)));
    return PDone.in(dataToWrite.getPipeline());
  } else {
    return dataToWrite.apply(TextIO.write().to(path));
  }
}
 
Example 21
Source Project: gcp-ingestion   Source File: RepublishPerChannel.java    License: Mozilla Public License 2.0 5 votes vote down vote up
@Override
public PDone expand(PCollection<PubsubMessage> input) {
  List<Destination> destinations = baseOptions.getPerChannelSampleRatios().entrySet().stream() //
      .map(Destination::new) //
      .collect(Collectors.toList());
  int numDestinations = destinations.size();
  int numPartitions = numDestinations + 1;
  PCollectionList<PubsubMessage> partitioned = input.apply("PartitionByChannel",
      Partition.of(numPartitions, new PartitionFn(destinations)));

  for (int i = 0; i < numDestinations; i++) {
    Destination destination = destinations.get(i);
    RepublisherOptions.Parsed opts = baseOptions.as(RepublisherOptions.Parsed.class);

    // The destination pattern here must be compile-time due to a detail of Dataflow's
    // streaming PubSub producer implementation; if that restriction is lifted in the future,
    // this can become a runtime parameter and we can perform replacement via NestedValueProvider.
    opts.setOutput(StaticValueProvider
        .of(baseOptions.getPerChannelDestination().replace("${channel}", destination.channel)));

    partitioned.get(i) //
        .apply("Sample" + destination.getCapitalizedChannel() + "BySampleIdOrRandomNumber",
            Filter.by(message -> {
              message = PubsubConstraints.ensureNonNull(message);
              String sampleId = message.getAttribute("sample_id");
              return RandomSampler.filterBySampleIdOrRandomNumber(sampleId, destination.ratio);
            }))
        .apply("Republish" + destination.getCapitalizedChannel() + "Sample",
            opts.getOutputType().write(opts));
  }

  return PDone.in(input.getPipeline());
}
 
Example 22
Source Project: gcp-ingestion   Source File: Write.java    License: Mozilla Public License 2.0 5 votes vote down vote up
@Override
public WithFailures.Result<PDone, PubsubMessage> expand(PCollection<PubsubMessage> input) {
  ValueProvider<DynamicPathTemplate> pathTemplate = NestedValueProvider.of(outputPrefix,
      DynamicPathTemplate::new);
  ValueProvider<String> staticPrefix = NestedValueProvider.of(pathTemplate,
      value -> value.staticPrefix);

  FileIO.Write<List<String>, PubsubMessage> write = FileIO
      .<List<String>, PubsubMessage>writeDynamic()
      // We can't pass the attribute map to by() directly since MapCoder isn't
      // deterministic;
      // instead, we extract an ordered list of the needed placeholder values.
      // That list is later available to withNaming() to determine output location.
      .by(message -> pathTemplate.get()
          .extractValuesFrom(DerivedAttributesMap.of(message.getAttributeMap())))
      .withDestinationCoder(ListCoder.of(StringUtf8Coder.of())) //
      .withCompression(compression) //
      .via(Contextful.fn(format::encodeSingleMessage), TextIO.sink()) //
      .to(staticPrefix) //
      .withNaming(placeholderValues -> NoColonFileNaming.defaultNaming(
          pathTemplate.get().replaceDynamicPart(placeholderValues), format.suffix()));

  if (inputType == InputType.pubsub) {
    // Passing a ValueProvider to withNumShards disables runner-determined sharding, so we
    // need to be careful to pass this only for streaming input (where runner-determined
    // sharding is not an option).
    write = write.withNumShards(numShards);
  }

  input //
      .apply(Window.<PubsubMessage>into(FixedWindows.of(windowDuration))
          // We allow lateness up to the maximum Cloud Pub/Sub retention of 7 days documented in
          // https://cloud.google.com/pubsub/docs/subscriber
          .withAllowedLateness(Duration.standardDays(7)) //
          .discardingFiredPanes())
      .apply(write);
  return WithFailures.Result.of(PDone.in(input.getPipeline()),
      EmptyErrors.in(input.getPipeline()));
}
 
Example 23
Source Project: gcp-ingestion   Source File: Write.java    License: Mozilla Public License 2.0 5 votes vote down vote up
@Override
public WithFailures.Result<PDone, PubsubMessage> expand(PCollection<PubsubMessage> input) {
  PDone done = input //
      .apply(CompressPayload.of(compression).withMaxCompressedBytes(maxCompressedBytes)) //
      .apply(PubsubConstraints.truncateAttributes()) //
      .apply(PubsubIO.writeMessages().to(topic));
  return WithFailures.Result.of(done, EmptyErrors.in(input.getPipeline()));
}
 
Example 24
Source Project: feast   Source File: WriteFeatureSetSpecAck.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public PDone expand(PCollection<FeatureSetReference> input) {
  return input
      .apply("Prepare", new PrepareWrite(getSinksCount()))
      .apply("FeatureSetSpecToAckMessage", ParDo.of(new BuildAckMessage()))
      .apply(
          "ToKafka",
          KafkaIO.<String, byte[]>write()
              .withBootstrapServers(
                  getSpecsStreamingUpdateConfig().getAck().getBootstrapServers())
              .withTopic(getSpecsStreamingUpdateConfig().getAck().getTopic())
              .withKeySerializer(StringSerializer.class)
              .withValueSerializer(ByteArraySerializer.class));
}
 
Example 25
Source Project: deployment-examples   Source File: WriteToBigQuery.java    License: MIT License 5 votes vote down vote up
@Override
public PDone expand(PCollection<InputT> teamAndScore) {
  teamAndScore
      .apply("ConvertToRow", ParDo.of(new BuildRowFn()))
      .apply(
          BigQueryIO.writeTableRows()
              .to(getTable(projectId, datasetId, tableName))
              .withSchema(getSchema())
              .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
              .withWriteDisposition(WriteDisposition.WRITE_APPEND));
  return PDone.in(teamAndScore.getPipeline());
}
 
Example 26
Source Project: beam   Source File: RedisIO.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public PDone expand(PCollection<KV<String, String>> input) {
  checkArgument(connectionConfiguration() != null, "withConnectionConfiguration() is required");

  input.apply(ParDo.of(new WriteFn(this)));
  return PDone.in(input.getPipeline());
}
 
Example 27
Source Project: deployment-examples   Source File: WriteToText.java    License: MIT License 5 votes vote down vote up
@Override
public PDone expand(PCollection<InputT> teamAndScore) {
  if (windowed) {
    teamAndScore
        .apply("ConvertToRow", ParDo.of(new BuildRowFn()))
        .apply(new WriteToText.WriteOneFilePerWindow(filenamePrefix));
  } else {
    teamAndScore
        .apply("ConvertToRow", ParDo.of(new BuildRowFn()))
        .apply(TextIO.write().to(filenamePrefix));
  }
  return PDone.in(teamAndScore.getPipeline());
}
 
Example 28
Source Project: deployment-examples   Source File: WriteOneFilePerWindow.java    License: MIT License 5 votes vote down vote up
@Override
public PDone expand(PCollection<String> input) {
  ResourceId resource = FileBasedSink.convertToFileResourceIfPossible(filenamePrefix);
  TextIO.Write write =
      TextIO.write()
          .to(new PerWindowFiles(resource))
          .withTempDirectory(resource.getCurrentDirectory())
          .withWindowedWrites();
  if (numShards != null) {
    write = write.withNumShards(numShards);
  }
  return input.apply(write);
}
 
Example 29
private static PDone buildIngestionPipeline(
    String transformPrefix,
    CdcApplierOptions options,
    PCollection<Row> input) {
  return input
      .apply(String.format("%s/ApplyChangesToBigQuery", transformPrefix),
          BigQueryChangeApplier.of(
              options.getChangeLogDataset(),
              options.getReplicaDataset(),
              options.getUpdateFrequencySecs(),
              options.as(GcpOptions.class).getProject()));
}
 
Example 30
Source Project: DataflowTemplates   Source File: WriteToGCSText.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public PDone expand(PCollection<KV<String, String>> kafkaRecords) {
  return kafkaRecords
      /*
       * Converting KV<String, String> records to String using DoFn.
       */
      .apply(
          "Converting to String",
          ParDo.of(
              new DoFn<KV<String, String>, String>() {
                @ProcessElement
                public void processElement(ProcessContext c) {
                  c.output(c.element().getValue());
                }
              }))
      /*
       * Writing as text file using {@link TextIO}.
       *
       * The {@link WindowedFilenamePolicy} class specifies the file path for writing the file.
       * The {@link withNumShards} option specifies the number of shards passed by the user.
       * The {@link withTempDirectory} option sets the base directory used to generate temporary files.
       */
      .apply(
          "Writing as Text",
          TextIO.write()
              .to(
                  new WindowedFilenamePolicy(
                      outputDirectory(),
                      outputFilenamePrefix(),
                      WriteToGCSUtility.SHARD_TEMPLATE,
                      WriteToGCSUtility.FILE_SUFFIX_MAP.get(WriteToGCSUtility.FileFormat.TEXT)))
              .withTempDirectory(
                  FileBasedSink.convertToFileResourceIfPossible(tempLocation())
                      .getCurrentDirectory())
              .withWindowedWrites()
              .withNumShards(numShards()));
}