com.hazelcast.jet.config.JobConfig Java Examples

public static void main(String[] args) {
    if (FlightDataSource.API_AUTHENTICATION_KEY.equals("YOUR_API_KEY_HERE")) {
         System.err.println("API_AUTHENTICATION_KEY not set in FlightDataSource.java");
         System.exit(1);
    }

    JetInstance jet = getJetInstance();

    Pipeline pipeline = buildPipeline();
    addListener(jet.getMap(TAKE_OFF_MAP), a -> System.out.println("New aircraft taking off: " + a));
    addListener(jet.getMap(LANDING_MAP), a -> System.out.println("New aircraft landing " + a));

    try {
        Job job = jet.newJob(pipeline, new JobConfig().setName("FlightTelemetry").setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE));
        job.join();
    } finally {
        Jet.shutdownAll();
    }
}
 

/**
 * <p>Run one copy of the Moving Average job
 * in this cluster.
 * </p>
 */
@Override
public void run(String... args) throws Exception {
	String prefix = this.getClass().getSimpleName() + " -";
	
	Pipeline pipeline = MovingAverage.build();
   		JobConfig jobConfig = new JobConfig();
   		jobConfig.setName(MyConstants.JOB_NAME);

   		// Run job if not already present
	Job job = this.jetInstance.getJob(jobConfig.getName());
   		if (job == null) {
       	    job = this.jetInstance.newJobIfAbsent(pipeline, jobConfig);
   		}

   		log.info("{} Job '{}', status '{}'.",
           		prefix, job.getName(), job.getStatus());
}
 

public static void main(String[] args) {
    validateWebcam();
    if (args.length != 1) {
        System.err.println("Missing command-line argument: <model path>");
        System.exit(1);
    }

    Path modelPath = Paths.get(args[0]).toAbsolutePath();
    if (!Files.isDirectory(modelPath)) {
        System.err.println("Model path does not exist (" + modelPath + ")");
        System.exit(1);
    }

    Pipeline pipeline = buildPipeline();

    JobConfig jobConfig = new JobConfig();
    jobConfig.attachDirectory(modelPath.toString(), "model");

    JetInstance jet = Jet.newJetInstance();
    try {
        jet.newJob(pipeline, jobConfig).join();
    } finally {
        Jet.shutdownAll();
    }
}
 

public static void main(String[] args) {
    System.setProperty("hazelcast.logging.type", "log4j");

    if (args.length != 2) {
        System.out.println("Usage: ModelServerClassification <data path> <model server address>");
        System.exit(1);
    }
    String dataPath = args[0];
    String serverAddress = args[1];

    JobConfig jobConfig = new JobConfig();
    jobConfig.attachDirectory(dataPath, "data");

    JetInstance instance = Jet.newJetInstance();
    try {
        IMap<Long, String> reviewsMap = instance.getMap("reviewsMap");
        SampleReviews.populateReviewsMap(reviewsMap);

        Pipeline p = buildPipeline(serverAddress, reviewsMap);

        instance.newJob(p, jobConfig).join();
    } finally {
        instance.shutdown();
    }
}
 

public static void main(String[] args) {
    System.setProperty("hazelcast.logging.type", "log4j");

    if (args.length != 1) {
        System.out.println("Usage: InProcessClassification <data path>");
        System.exit(1);
    }

    String dataPath = args[0];
    JetInstance instance = Jet.newJetInstance();
    JobConfig jobConfig = new JobConfig();
    jobConfig.attachDirectory(dataPath, "data");

    try {
        IMap<Long, String> reviewsMap = instance.getMap("reviewsMap");
        SampleReviews.populateReviewsMap(reviewsMap);
        instance.newJob(buildPipeline(reviewsMap), jobConfig).join();
    } finally {
        instance.shutdown();
    }
}
 

public static void main(String[] args) {
    Pipeline p = buildPipeline();

    JetInstance jet = Jet.bootstrappedInstance();

    try {
        JobConfig jobConfig = new JobConfig()
                .setAutoScaling(true)
                .setName("TradeAnalysis")
                .setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE);

        jet.newJob(p, jobConfig).join();
    } finally {
        Jet.shutdownAll();
    }
}
 

private JobConfig getJobConfig(JetPipelineOptions options) {
  JobConfig jobConfig = new JobConfig();

  String jobName = options.getJobName();
  if (jobName != null) {
    jobConfig.setName(jobName);
  }

  boolean hasNoLocalMembers = options.getJetLocalMode() <= 0;
  if (hasNoLocalMembers) {
    String codeJarPathname = options.getCodeJarPathname();
    if (codeJarPathname != null && !codeJarPathname.isEmpty()) {
      jobConfig.addJar(codeJarPathname);
    }
  }

  return jobConfig;
}
 

@Test
public void readFromRandomSource() throws Exception {
    System.setProperty("hazelcast.logging.type", "log4j");
    Properties randomProperties = new Properties();
    randomProperties.setProperty("name", "random-source-connector");
    randomProperties.setProperty("connector.class", "sasakitoa.kafka.connect.random.RandomSourceConnector");
    randomProperties.setProperty("generator.class", "sasakitoa.kafka.connect.random.generator.RandomInt");
    randomProperties.setProperty("tasks.max", "1");
    randomProperties.setProperty("messages.per.second", "1000");
    randomProperties.setProperty("topic", "test");
    randomProperties.setProperty("task.summary.enable", "true");

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(KafkaConnectSources.connect(randomProperties))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> assertEquals(ITEM_COUNT, list.size())));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJar(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("random-connector-1.0-SNAPSHOT.jar"))
                                  .getPath()
    );

    Job job = createJetMember().newJob(pipeline, jobConfig);

    sleepAtLeastSeconds(5);

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

public static void main(String[] args) {
    if (args.length != 2) {
        System.err.println("Missing command-line arguments: <model-file-path> <validation-input-data>");
        System.exit(1);
    }

    Path modelFile = Paths.get(args[0]).toAbsolutePath();
    Path inputFile = Paths.get(args[1]).toAbsolutePath();
    validateFileReadable(modelFile);
    validateFileReadable(inputFile);

    System.setProperty("hazelcast.logging.type", "log4j");

    JetInstance jet = Jet.newJetInstance();

    JobConfig jobConfig = new JobConfig();
    jobConfig.setName("h2o Breast Cancer Classification");
    jobConfig.attachFile(modelFile.toString(), "model");

    Job job = jet.newJob(buildPipeline(inputFile), jobConfig);

    try {
        job.join();
    } finally {
        jet.shutdown();
    }
}
 

@Override
public void run(String... args) throws Exception {
	Pipeline pipeline = FileWatcher.build();
	
	JobConfig jobConfig = new JobConfig();
	jobConfig.setName(FileWatcher.class.getSimpleName());

	this.jetInstance.newJobIfAbsent(pipeline, jobConfig);
}
 

@Test
public void readFromPostgres() throws Exception {
    Configuration configuration = Configuration
            .create()
            .with("name", "postgres-inventory-connector")
            .with("connector.class", "io.debezium.connector.postgresql.PostgresConnector")
            /* begin connector properties */
            .with("tasks.max", "1")
            .with("database.hostname", postgres.getContainerIpAddress())
            .with("database.port", postgres.getMappedPort(POSTGRESQL_PORT))
            .with("database.user", "postgres")
            .with("database.password", "postgres")
            .with("database.dbname", "postgres")
            .with("database.server.name", "dbserver1")
            .with("schema.whitelist", "inventory")
            .with("database.history.hazelcast.list.name", "test")
            .build();

    JetInstance jet = createJetMember();

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(DebeziumSources.cdc(configuration))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> assertTrue(list.stream().anyMatch(s -> s.contains("Anne Marie")))));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJarsInZip(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("debezium-connector-postgres.zip")));

    Job job = jet.newJob(pipeline, jobConfig);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    // update record
    try (Connection connection = DriverManager.getConnection(postgres.getJdbcUrl(), postgres.getUsername(),
            postgres.getPassword())) {
        connection.setSchema("inventory");
        PreparedStatement preparedStatement = connection.prepareStatement("update customers set " +
                "first_name = 'Anne Marie' where id = 1004;");
        preparedStatement.executeUpdate();
    }

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }


}
 

@Test
public void readFromMySql() throws Exception {
    // given
    Configuration configuration = Configuration
            .create()
            .with("name", "mysql-inventory-connector")
            .with("connector.class", "io.debezium.connector.mysql.MySqlConnector")
            /* begin connector properties */
            .with("database.hostname", mysql.getContainerIpAddress())
            .with("database.port", mysql.getMappedPort(MYSQL_PORT))
            .with("database.user", "debezium")
            .with("database.password", "dbz")
            .with("database.server.id", "184054")
            .with("database.server.name", "dbserver1")
            .with("database.whitelist", "inventory")
            .with("table.whitelist", "inventory.customers")
            .with("include.schema.changes", "false")
            .with("database.history.hazelcast.list.name", "test")
            .build();

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(DebeziumSources.cdc(configuration))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .filterUsingService(ServiceFactories.sharedService(context -> {
                Serde<EventRecord> serde = DebeziumSerdes.payloadJson(EventRecord.class);
                serde.configure(Collections.emptyMap(), false);
                return serde.deserializer();
            }, Deserializer::close), (deserializer, record) -> {
                EventRecord eventRecord = deserializer.deserialize("", record.getBytes());
                return eventRecord.isUpdate();
            })
            .writeTo(AssertionSinks.assertCollectedEventually(30,
                    list -> Assert.assertTrue(list.stream().anyMatch(s -> s.contains("Anne Marie")))));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJarsInZip(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("debezium-connector-mysql.zip")));

    // when
    JetInstance jet = createJetMember();
    Job job = jet.newJob(pipeline, jobConfig);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    sleepAtLeastSeconds(10);
    // update a record
    try (Connection connection = DriverManager.getConnection(mysql.withDatabaseName("inventory").getJdbcUrl(),
            mysql.getUsername(), mysql.getPassword())) {
        PreparedStatement preparedStatement = connection
                .prepareStatement("UPDATE customers SET first_name='Anne Marie' WHERE id=1004;");
        preparedStatement.executeUpdate();
    }


    // then
    try {
        job.join();
        Assert.fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        Assert.assertTrue("Job was expected to complete with " +
                        "AssertionCompletedException, but completed with: " + e.getCause(),
                errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

@Test
public void readFromMsSql() throws Exception {
    execInContainer("setup.sql");
    assertTrueEventually(() -> {
        Container.ExecResult result = execInContainer("cdc.sql");
        assertContains(result.getStdout(), "already");
    });

    Configuration configuration = Configuration
            .create()
            .with("name", "mssql-inventory-connector")
            .with("connector.class", "io.debezium.connector.sqlserver.SqlServerConnector")
            /* begin connector properties */
            .with("tasks.max", "1")
            .with("database.hostname", mssql.getContainerIpAddress())
            .with("database.port", mssql.getMappedPort(MS_SQL_SERVER_PORT))
            .with("database.user", mssql.getUsername())
            .with("database.password", mssql.getPassword())
            .with("database.dbname", "MyDB")
            .with("database.server.name", "fulfillment")
            .with("table.whitelist", "inventory.customers")
            .with("database.history.hazelcast.list.name", "test")
            .build();

    JetInstance jet = createJetMember();

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(DebeziumSources.cdc(configuration))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> assertTrue(list.stream().anyMatch(s -> s.contains("Anne Marie")))));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJarsInZip(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("debezium-connector-sqlserver.zip")));

    Job job = jet.newJob(pipeline, jobConfig);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    sleepAtLeastSeconds(30);
    // update record
    try (Connection connection = DriverManager.getConnection(mssql.getJdbcUrl() + ";databaseName=MyDB",
            mssql.getUsername(), mssql.getPassword())) {
        connection.setSchema("inventory");
        PreparedStatement preparedStatement = connection.prepareStatement("update MyDB.inventory.customers set " +
                "first_name = 'Anne Marie' where id = 1001;");
        preparedStatement.executeUpdate();
    }

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }


}
 

@Test
public void readFromMongoDb() throws Exception {
    // populate initial data
    mongo.execInContainer("sh", "-c", "/usr/local/bin/init-inventory.sh");

    Configuration configuration = Configuration
            .create()
            .with("name", "mongodb-inventory-connector")
            .with("connector.class", "io.debezium.connector.mongodb.MongoDbConnector")
            /* begin connector properties */
            .with("mongodb.hosts", "rs0/" + mongo.getContainerIpAddress() + ":"
                    + mongo.getMappedPort(MongoDBContainer.MONGODB_PORT))
            .with("mongodb.name", "fullfillment")
            .with("mongodb.user", "debezium")
            .with("mongodb.password", "dbz")
            .with("mongodb.members.auto.discover", "false")
            .with("collection.whitelist", "inventory.*")
            .with("database.history.hazelcast.list.name", "test")
            .build();

    JetInstance jet = createJetMember();

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(DebeziumSources.cdc(configuration))
            .withoutTimestamps()
            .map(record -> Values.convertToString(record.valueSchema(), record.value()))
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> Assert.assertTrue(list.stream().anyMatch(s -> s.contains("Jason")))));

    JobConfig jobConfig = new JobConfig();
    jobConfig.addJarsInZip(Objects.requireNonNull(this.getClass()
                                                      .getClassLoader()
                                                      .getResource("debezium-connector-mongodb.zip")));

    Job job = jet.newJob(pipeline, jobConfig);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    // update record
    mongo.execInContainer("sh", "-c", "/usr/local/bin/insertData.sh");

    try {
        job.join();
        Assert.fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        Assert.assertTrue("Job was expected to complete with AssertionCompletedException, " +
                "but completed with: " + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }


}