Java Code Examples for org.apache.beam.sdk.options.PipelineOptionsFactory#register()

The following examples show how to use org.apache.beam.sdk.options.PipelineOptionsFactory#register() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MongoDBIOIT.java    From beam with Apache License 2.0 7 votes vote down vote up 
@BeforeClass
public static void setUp() {
  PipelineOptionsFactory.register(MongoDBPipelineOptions.class);
  options = TestPipeline.testingPipelineOptions().as(MongoDBPipelineOptions.class);
  collection = String.format("test_%s", new Date().getTime());
  bigQueryDataset = options.getBigQueryDataset();
  bigQueryTable = options.getBigQueryTable();
  mongoUrl =
      String.format("mongodb://%s:%s", options.getMongoDBHostName(), options.getMongoDBPort());
  mongoClient = MongoClients.create(mongoUrl);
  settings =
      InfluxDBSettings.builder()
          .withHost(options.getInfluxHost())
          .withDatabase(options.getInfluxDatabase())
          .withMeasurement(options.getInfluxMeasurement())
          .get();
}
Example 2
Source File: BigtableWriteIT.java    From beam with Apache License 2.0 6 votes vote down vote up 
@Before
public void setup() throws Exception {
  PipelineOptionsFactory.register(BigtableTestOptions.class);
  options = TestPipeline.testingPipelineOptions().as(BigtableTestOptions.class);
  project = options.as(GcpOptions.class).getProject();

  bigtableOptions =
      new Builder()
          .setProjectId(project)
          .setInstanceId(options.getInstanceId())
          .setUserAgent("apache-beam-test")
          .build();

  session =
      new BigtableSession(
          bigtableOptions
              .toBuilder()
              .setCredentialOptions(
                  CredentialOptions.credential(options.as(GcpOptions.class).getGcpCredential()))
              .build());
  tableAdminClient = session.getTableAdminClient();
}
Example 3
Source File: ElasticsearchIOIT.java    From beam with Apache License 2.0 6 votes vote down vote up 
@BeforeClass
public static void beforeClass() throws Exception {
  PipelineOptionsFactory.register(ElasticsearchPipelineOptions.class);
  options = TestPipeline.testingPipelineOptions().as(ElasticsearchPipelineOptions.class);
  readConnectionConfiguration =
      ElasticsearchIOITCommon.getConnectionConfiguration(
          options, ElasticsearchIOITCommon.IndexMode.READ);
  writeConnectionConfiguration =
      ElasticsearchIOITCommon.getConnectionConfiguration(
          options, ElasticsearchIOITCommon.IndexMode.WRITE);
  updateConnectionConfiguration =
      ElasticsearchIOITCommon.getConnectionConfiguration(
          options, ElasticsearchIOITCommon.IndexMode.WRITE_PARTIAL);
  restClient = readConnectionConfiguration.createClient();
  elasticsearchIOTestCommon =
      new ElasticsearchIOTestCommon(readConnectionConfiguration, restClient, true);
}
Example 4
Source File: V1WriteIT.java    From beam with Apache License 2.0 5 votes vote down vote up 
@Before
public void setup() {
  PipelineOptionsFactory.register(V1TestOptions.class);
  options = TestPipeline.testingPipelineOptions().as(V1TestOptions.class);
  project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject();
  ancestor = UUID.randomUUID().toString();
}
Example 5
Source File: SpannerReadIT.java    From beam with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() throws Exception {
  PipelineOptionsFactory.register(SpannerTestPipelineOptions.class);
  options = TestPipeline.testingPipelineOptions().as(SpannerTestPipelineOptions.class);

  project = options.getInstanceProjectId();
  if (project == null) {
    project = options.as(GcpOptions.class).getProject();
  }

  spanner = SpannerOptions.newBuilder().setProjectId(project).build().getService();

  databaseName = generateDatabaseName();

  databaseAdminClient = spanner.getDatabaseAdminClient();

  // Delete database if exists.
  databaseAdminClient.dropDatabase(options.getInstanceId(), databaseName);

  OperationFuture<Database, CreateDatabaseMetadata> op =
      databaseAdminClient.createDatabase(
          options.getInstanceId(),
          databaseName,
          Collections.singleton(
              "CREATE TABLE "
                  + options.getTable()
                  + " ("
                  + "  Key           INT64,"
                  + "  Value         STRING(MAX),"
                  + ") PRIMARY KEY (Key)"));
  op.get();
  makeTestData();
}
Example 6
Source File: BigQueryIOStorageQueryIT.java    From beam with Apache License 2.0 5 votes vote down vote up 
private void setUpTestEnvironment(String tableSize) {
  PipelineOptionsFactory.register(BigQueryIOStorageQueryOptions.class);
  options = TestPipeline.testingPipelineOptions().as(BigQueryIOStorageQueryOptions.class);
  options.setNumRecords(EXPECTED_NUM_RECORDS.get(tableSize));
  String project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject();
  options.setInputTable(project + '.' + DATASET_ID + '.' + TABLE_PREFIX + tableSize);
}
Example 7
Source File: BigQueryTimePartitioningClusteringIT.java    From beam with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() {
  PipelineOptionsFactory.register(BigQueryClusteringITOptions.class);
  options = TestPipeline.testingPipelineOptions().as(BigQueryClusteringITOptions.class);
  options.setTempLocation(options.getTempRoot() + "/temp-it/");
  bqClient = BigqueryClient.getNewBigquerryClient(options.getAppName());
}
Example 8
Source File: SpannerWriteIT.java    From beam with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() throws Exception {
  PipelineOptionsFactory.register(SpannerTestPipelineOptions.class);
  options = TestPipeline.testingPipelineOptions().as(SpannerTestPipelineOptions.class);

  project = options.getInstanceProjectId();
  if (project == null) {
    project = options.as(GcpOptions.class).getProject();
  }

  spanner = SpannerOptions.newBuilder().setProjectId(project).build().getService();

  databaseName = generateDatabaseName();

  databaseAdminClient = spanner.getDatabaseAdminClient();

  // Delete database if exists.
  databaseAdminClient.dropDatabase(options.getInstanceId(), databaseName);

  OperationFuture<Database, CreateDatabaseMetadata> op =
      databaseAdminClient.createDatabase(
          options.getInstanceId(),
          databaseName,
          Collections.singleton(
              "CREATE TABLE "
                  + options.getTable()
                  + " ("
                  + "  Key           INT64,"
                  + "  Value         STRING(MAX) NOT NULL,"
                  + ") PRIMARY KEY (Key)"));
  op.get();
}
Example 9
Source File: InputAvroSchemaTest.java    From dbeam with Apache License 2.0 5 votes vote down vote up 
private QueryBuilderArgs pareOptions(String cmdLineArgs) throws IOException {
  PipelineOptionsFactory.register(JdbcExportPipelineOptions.class);
  JdbcExportPipelineOptions opts =
      PipelineOptionsFactory.fromArgs(cmdLineArgs.split(" "))
          .withValidation()
          .create()
          .as(JdbcExportPipelineOptions.class);
  return JdbcExportArgsFactory.createQueryArgs(opts);
}
Example 10
Source File: BigQueryIOReadIT.java    From beam with Apache License 2.0 5 votes vote down vote up 
private void setupTestEnvironment(String recordSize) {
  PipelineOptionsFactory.register(BigQueryIOReadOptions.class);
  options = TestPipeline.testingPipelineOptions().as(BigQueryIOReadOptions.class);
  options.setNumRecords(numOfRecords.get(recordSize));
  options.setTempLocation(options.getTempRoot() + "/temp-it/");
  project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject();
  options.setInputTable(project + ":" + datasetId + "." + tablePrefix + recordSize);
}
Example 11
Source File: KinesisIOIT.java    From beam with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void setup() {
  PipelineOptionsFactory.register(KinesisTestOptions.class);
  options = TestPipeline.testingPipelineOptions().as(KinesisTestOptions.class);
  numberOfShards = options.getNumberOfShards();
  numberOfRows = options.getNumberOfRecords();
}
Example 12
Source File: JdbcExportOptionsTest.java    From dbeam with Apache License 2.0 4 votes vote down vote up 
JdbcExportArgs optionsFromArgs(String[] cmdLineArgs) throws IOException, ClassNotFoundException {
  PipelineOptionsFactory.register(JdbcExportPipelineOptions.class);
  PipelineOptions opts = PipelineOptionsFactory.fromArgs(cmdLineArgs).withValidation().create();
  return JdbcExportArgsFactory.fromPipelineOptions(opts);
}
Example 13
Source File: WindowedWordCountIT.java    From beam with Apache License 2.0 4 votes vote down vote up 
@BeforeClass
public static void setUp() {
  PipelineOptionsFactory.register(TestPipelineOptions.class);
}
Example 14
Source File: TfIdfIT.java    From beam with Apache License 2.0 4 votes vote down vote up 
@BeforeClass
public static void setUp() {
  PipelineOptionsFactory.register(TfIdfITOptions.class);
}
Example 15
Source File: HadoopFormatIOElasticIT.java    From beam with Apache License 2.0 4 votes vote down vote up 
@BeforeClass
public static void setUp() {
  PipelineOptionsFactory.register(HadoopFormatIOTestOptions.class);
  options = TestPipeline.testingPipelineOptions().as(HadoopFormatIOTestOptions.class);
}
Example 16
Source File: BigQueryTornadoesIT.java    From beam with Apache License 2.0 4 votes vote down vote up 
@BeforeClass
public static void setUp() {
  PipelineOptionsFactory.register(BigQueryTornadoesITOptions.class);
}
Example 17
Source File: HadoopFormatIOCassandraIT.java    From beam with Apache License 2.0 4 votes vote down vote up 
@BeforeClass
public static void setUp() {
  PipelineOptionsFactory.register(HadoopFormatIOTestOptions.class);
  options = TestPipeline.testingPipelineOptions().as(HadoopFormatIOTestOptions.class);
}
Example 18
Source File: RequiresStableInputIT.java    From beam with Apache License 2.0 4 votes vote down vote up 
@BeforeClass
public static void setup() {
  PipelineOptionsFactory.register(TestPipelineOptions.class);
}
Example 19
Source File: ElasticsearchIOITCommon.java    From beam with Apache License 2.0 3 votes vote down vote up 
/**
 * Use this to create the index for reading before IT read tests.
 *
 * <p>To invoke this class, you can use this command line from elasticsearch-tests-common module
 * directory after setting the correct server IP:
 *
 * <pre>
 * mvn test-compile exec:java \
 * -Dexec.mainClass=org.apache.beam.sdk.io.elasticsearch.ElasticsearchIOITCommon \
 * -Dexec.args="--elasticsearchServer=127.0.0.1 \
 * --elasticsearchHttpPort=9200" \
 * -Dexec.classpathScope=test
 *   </pre>
 *
 * @param args Please pass options from ElasticsearchTestOptions used for connection to
 *     Elasticsearch as shown above.
 */
public static void main(String[] args) throws Exception {
  PipelineOptionsFactory.register(ElasticsearchPipelineOptions.class);
  ElasticsearchPipelineOptions options =
      PipelineOptionsFactory.fromArgs(args).as(ElasticsearchPipelineOptions.class);
  createAndPopulateReadIndex(options);
}
Example 20
Source File: IOITHelper.java    From beam with Apache License 2.0 3 votes vote down vote up 
public static <T extends PipelineOptions> T readIOTestPipelineOptions(Class<T> optionsType) {

    PipelineOptionsFactory.register(optionsType);
    PipelineOptions options = TestPipeline.testingPipelineOptions().as(optionsType);

    return PipelineOptionsValidator.validate(optionsType, options);
  }