org.apache.flink.table.descriptors.Json Java Examples

The following examples show how to use org.apache.flink.table.descriptors.Json. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: JsonRowFormatFactoryTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testSchema() {
	final Map<String, String> properties = toMap(
		new Json()
			.schema(SCHEMA)
			.failOnMissingField(false));

	testSchemaSerializationSchema(properties);

	testSchemaDeserializationSchema(properties);
}

Example #2

Source File: JsonRowFormatFactoryTest.java From flink with Apache License 2.0

5 votes

@Test
public void testSchemaDerivationByDefault() {
	final Map<String, String> properties = toMap(
		new Schema()
			.field("field1", Types.BOOLEAN())
			.field("field2", Types.INT())
			.field("proctime", Types.SQL_TIMESTAMP()).proctime(),
		new Json());

	testSchemaSerializationSchema(properties);

	testSchemaDeserializationSchema(properties);
}

Example #3

Source File: JsonRowFormatFactoryTest.java From flink with Apache License 2.0

5 votes

@Test
public void testSchemaDerivation() {
	final Map<String, String> properties = toMap(
		new Schema()
			.field("field1", Types.BOOLEAN())
			.field("field2", Types.INT())
			.field("proctime", Types.SQL_TIMESTAMP()).proctime(),
		new Json()
			.deriveSchema());

	testSchemaSerializationSchema(properties);

	testSchemaDeserializationSchema(properties);
}

Example #4

Source File: JsonRowFormatFactoryTest.java From flink with Apache License 2.0

5 votes

@Test
public void testJsonSchema() {
	final Map<String, String> properties = toMap(
		new Json()
			.jsonSchema(JSON_SCHEMA)
			.failOnMissingField(true));

	testJsonSchemaSerializationSchema(properties);

	testJsonSchemaDeserializationSchema(properties);
}

Example #5

Source File: JsonRowFormatFactoryTest.java From flink with Apache License 2.0

5 votes

@Test
public void testSchema() {
	final Map<String, String> properties = toMap(
		new Json()
			.schema(SCHEMA)
			.failOnMissingField(false));

	testSchemaSerializationSchema(properties);

	testSchemaDeserializationSchema(properties);
}

Example #6

Source File: ElasticsearchUpsertTableSinkFactoryTestBase.java From flink with Apache License 2.0

5 votes

protected Map<String, String> createElasticSearchProperties() {
	return new TestTableDescriptor(
		new Elasticsearch()
			.version(getElasticsearchVersion())
			.host(HOSTNAME, PORT, SCHEMA)
			.index(INDEX)
			.documentType(DOC_TYPE)
			.keyDelimiter(KEY_DELIMITER)
			.keyNullLiteral(KEY_NULL_LITERAL)
			.bulkFlushBackoffExponential()
			.bulkFlushBackoffDelay(123L)
			.bulkFlushBackoffMaxRetries(3)
			.bulkFlushInterval(100L)
			.bulkFlushMaxActions(1000)
			.bulkFlushMaxSize("1 MB")
			.failureHandlerCustom(DummyFailureHandler.class)
			.connectionMaxRetryTimeout(100)
			.connectionPathPrefix("/myapp"))
		.withFormat(
			new Json()
				.deriveSchema())
		.withSchema(
			new Schema()
				.field(FIELD_KEY, DataTypes.BIGINT())
				.field(FIELD_FRUIT_NAME, DataTypes.STRING())
				.field(FIELD_COUNT, DataTypes.DECIMAL(10, 4))
				.field(FIELD_TS, DataTypes.TIMESTAMP(3)))
		.inUpsertMode()
		.toProperties();
}

Example #7

Source File: FlinkTableITCase.java From flink-connectors with Apache License 2.0

5 votes

@Test
public void testBatchTableSinkUsingDescriptor() throws Exception {

    // create a Pravega stream for test purposes
    Stream stream = Stream.of(setupUtils.getScope(), "testBatchTableSinkUsingDescriptor");
    this.setupUtils.createTestStream(stream.getStreamName(), 1);

    // create a Flink Table environment
    ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(1);
    BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env);

    Table table = tableEnv.fromDataSet(env.fromCollection(SAMPLES));

    Pravega pravega = new Pravega();
    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("category")
            .forStream(stream)
            .withPravegaConfig(setupUtils.getPravegaConfig());

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(new Json().failOnMissingField(true))
            .withSchema(new Schema().field("category", DataTypes.STRING()).
                    field("value", DataTypes.INT()));
    desc.createTemporaryTable("test");

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSink<?> sink = TableFactoryService.find(BatchTableSinkFactory.class, propertiesMap)
            .createBatchTableSink(propertiesMap);

    String tableSinkPath = tableEnv.getCurrentDatabase() + "." + "PravegaSink";

    ConnectorCatalogTable<?, ?> connectorCatalogSinkTable = ConnectorCatalogTable.sink(sink, true);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tableSinkPath),
            connectorCatalogSinkTable, false);
    table.insertInto("PravegaSink");
    env.execute();
}

Example #8

Source File: JsonRowFormatFactoryTest.java From flink with Apache License 2.0

5 votes

@Test
public void testSchemaDerivation() {
	final Map<String, String> properties = toMap(
		new Schema()
			.field("field1", Types.BOOLEAN())
			.field("field2", Types.INT())
			.field("proctime", Types.SQL_TIMESTAMP()).proctime(),
		new Json()
			.deriveSchema());

	testSchemaSerializationSchema(properties);

	testSchemaDeserializationSchema(properties);
}

Example #9

Source File: JsonRowFormatFactoryTest.java From flink with Apache License 2.0

5 votes

@Test
public void testJsonSchema() {
	final Map<String, String> properties = toMap(
		new Json()
			.jsonSchema(JSON_SCHEMA)
			.failOnMissingField(true));

	testJsonSchemaSerializationSchema(properties);

	testJsonSchemaDeserializationSchema(properties);
}

Example #10

Source File: JsonRowFormatFactoryTest.java From flink with Apache License 2.0

5 votes

@Test
public void testSchema() {
	final Map<String, String> properties = toMap(
		new Json()
			.schema(SCHEMA)
			.failOnMissingField(false));

	testSchemaSerializationSchema(properties);

	testSchemaDeserializationSchema(properties);
}

Example #11

Source File: JsonRowFormatFactoryTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testSchemaDerivation() {
	final Map<String, String> properties = toMap(
		new Schema()
			.field("field1", Types.BOOLEAN())
			.field("field2", Types.INT())
			.field("proctime", Types.SQL_TIMESTAMP()).proctime(),
		new Json()
			.deriveSchema());

	testSchemaSerializationSchema(properties);

	testSchemaDeserializationSchema(properties);
}

Example #12

Source File: JsonRowFormatFactoryTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testJsonSchema() {
	final Map<String, String> properties = toMap(
		new Json()
			.jsonSchema(JSON_SCHEMA)
			.failOnMissingField(true));

	testJsonSchemaSerializationSchema(properties);

	testJsonSchemaDeserializationSchema(properties);
}

Example #13

Source File: FlinkPravegaTableITCase.java From flink-connectors with Apache License 2.0

4 votes

private void testTableSourceStreamingDescriptor(Stream stream, PravegaConfig pravegaConfig) throws Exception {
    final StreamExecutionEnvironment execEnvRead = StreamExecutionEnvironment.getExecutionEnvironment();
    execEnvRead.setParallelism(1);
    execEnvRead.enableCheckpointing(100);
    execEnvRead.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

    StreamTableEnvironment tableEnv = StreamTableEnvironment.create(execEnvRead,
            EnvironmentSettings.newInstance()
                    // watermark is only supported in blink planner
                    .useBlinkPlanner()
                    .inStreamingMode()
                    .build());
    RESULTS.clear();

    // read data from the stream using Table reader
    Schema schema = new Schema()
            .field("user", DataTypes.STRING())
            .field("uri", DataTypes.STRING())
            .field("accessTime", DataTypes.TIMESTAMP(3)).rowtime(
                    new Rowtime().timestampsFromField("accessTime").watermarksPeriodicBounded(30000L));

    Pravega pravega = new Pravega();
    pravega.tableSourceReaderBuilder()
            .withReaderGroupScope(stream.getScope())
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(new Json().failOnMissingField(true))
            .withSchema(schema)
            .inAppendMode();

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSource<?> source = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
            .createStreamTableSource(propertiesMap);

    String tableSourcePath = tableEnv.getCurrentDatabase() + "." + "MyTableRow";

    ConnectorCatalogTable<?, ?> connectorCatalogSourceTable = ConnectorCatalogTable.source(source, false);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tableSourcePath),
            connectorCatalogSourceTable, false);

    String sqlQuery = "SELECT user, " +
            "TUMBLE_END(accessTime, INTERVAL '5' MINUTE) AS accessTime, " +
            "COUNT(uri) AS cnt " +
            "from MyTableRow GROUP BY " +
            "user, TUMBLE(accessTime, INTERVAL '5' MINUTE)";
    Table result = tableEnv.sqlQuery(sqlQuery);

    DataStream<Tuple2<Boolean, Row>> resultSet = tableEnv.toRetractStream(result, Row.class);
    StringSink2 stringSink = new StringSink2(8);
    resultSet.addSink(stringSink);

    try {
        execEnvRead.execute("ReadRowData");
    } catch (Exception e) {
        if (!(ExceptionUtils.getRootCause(e) instanceof SuccessException)) {
            throw e;
        }
    }

    log.info("results: {}", RESULTS);
    boolean compare = compare(RESULTS, getExpectedResultsAppend());
    assertTrue("Output does not match expected result", compare);
}

Example #14

Source File: FlinkPravegaTableITCase.java From flink-connectors with Apache License 2.0

4 votes

private void testTableSourceBatchDescriptor(Stream stream, PravegaConfig pravegaConfig) throws Exception {
    ExecutionEnvironment execEnvRead = ExecutionEnvironment.getExecutionEnvironment();
    // Can only use Legacy Flink planner for BatchTableEnvironment
    BatchTableEnvironment tableEnv = BatchTableEnvironment.create(execEnvRead);
    execEnvRead.setParallelism(1);

    Schema schema = new Schema()
            .field("user", DataTypes.STRING())
            .field("uri", DataTypes.STRING())
            // Note: LocalDateTime is not supported in legacy Flink planner, bridged to Timestamp with the data source.
            // See https://issues.apache.org/jira/browse/FLINK-16693 for more information.
            .field("accessTime", DataTypes.TIMESTAMP(3).bridgedTo(Timestamp.class));

    Pravega pravega = new Pravega();

    pravega.tableSourceReaderBuilder()
            .withReaderGroupScope(stream.getScope())
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(new Json().failOnMissingField(false))
            .withSchema(schema);

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSource<?> source = TableFactoryService.find(BatchTableSourceFactory.class, propertiesMap)
            .createBatchTableSource(propertiesMap);

    String tableSourcePath = tableEnv.getCurrentDatabase() + "." + "MyTableRow";

    ConnectorCatalogTable<?, ?> connectorCatalogSourceTable = ConnectorCatalogTable.source(source, true);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tableSourcePath),
            connectorCatalogSourceTable, false);

    String sqlQuery = "SELECT user, count(uri) from MyTableRow GROUP BY user";

    Table result = tableEnv.sqlQuery(sqlQuery);

    DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);

    List<Row> results = resultSet.collect();
    log.info("results: {}", results);

    boolean compare = compare(results, getExpectedResultsRetracted());
    assertTrue("Output does not match expected result", compare);
}

Example #15

Source File: FlinkPravegaTableSourceTest.java From flink-connectors with Apache License 2.0

4 votes

@Test
@SuppressWarnings("unchecked")
public void testTableSourceDescriptor() {
    final String cityName = "fruitName";
    final String total = "count";
    final String eventTime = "eventTime";
    final String procTime = "procTime";
    final String controllerUri = "tcp://localhost:9090";
    final long delay = 3000L;
    final String streamName = "test";
    final String scopeName = "test";

    final TableSchema tableSchema = TableSchema.builder()
            .field(cityName, DataTypes.STRING())
            .field(total, DataTypes.BIGINT())
            .field(eventTime, DataTypes.TIMESTAMP(3))
            .field(procTime, DataTypes.TIMESTAMP(3))
            .build();

    Stream stream = Stream.of(scopeName, streamName);
    PravegaConfig pravegaConfig = PravegaConfig.fromDefaults()
            .withControllerURI(URI.create(controllerUri))
            .withDefaultScope(scopeName);

    // construct table source using descriptors and table source factory
    Pravega pravega = new Pravega();
    pravega.tableSourceReaderBuilder()
            .withReaderGroupScope(stream.getScope())
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);

    final TestTableDescriptor testDesc = new TestTableDescriptor(pravega)
            .withFormat(new Json().failOnMissingField(false))
            .withSchema(
                    new Schema()
                            .field(cityName, DataTypes.STRING())
                            .field(total, DataTypes.BIGINT())
                            .field(eventTime, DataTypes.TIMESTAMP(3))
                                .rowtime(new Rowtime()
                                            .timestampsFromField(eventTime)
                                            .watermarksFromStrategy(new BoundedOutOfOrderTimestamps(delay))
                                        )
                            .field(procTime, DataTypes.TIMESTAMP(3)).proctime())
            .inAppendMode();

    final Map<String, String> propertiesMap = testDesc.toProperties();
    final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
            .createStreamTableSource(propertiesMap);
    assertNotNull(actualSource);
    TableSourceValidation.validateTableSource(actualSource, tableSchema);
}

Example #16

Source File: FlinkPravegaTableSourceTest.java From flink-connectors with Apache License 2.0

4 votes

@Test
@SuppressWarnings("unchecked")
public void testTableSourceDescriptorWithWatermark() {
    final String cityName = "fruitName";
    final String total = "count";
    final String eventTime = "eventTime";
    final String controllerUri = "tcp://localhost:9090";
    final String streamName = "test";
    final String scopeName = "test";

    Stream stream = Stream.of(scopeName, streamName);
    PravegaConfig pravegaConfig = PravegaConfig.fromDefaults()
            .withControllerURI(URI.create(controllerUri))
            .withDefaultScope(scopeName);

    // construct table source using descriptors and table source factory
    Pravega pravega = new Pravega();
    pravega.tableSourceReaderBuilder()
            .withTimestampAssigner(new MyAssigner())
            .withReaderGroupScope(stream.getScope())
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);

    final TableSchema tableSchema = TableSchema.builder()
            .field(cityName, DataTypes.STRING())
            .field(total, DataTypes.INT())
            .field(eventTime, DataTypes.TIMESTAMP(3))
            .build();

    final TestTableDescriptor testDesc = new TestTableDescriptor(pravega)
            .withFormat(new Json().failOnMissingField(false))
            .withSchema(
                    new Schema()
                            .field(cityName, DataTypes.STRING())
                            .field(total, DataTypes.INT())
                            .field(eventTime, DataTypes.TIMESTAMP(3))
                            .rowtime(new Rowtime()
                                    .timestampsFromSource()
                                    .watermarksFromSource()
                            ))
            .inAppendMode();

    final Map<String, String> propertiesMap = testDesc.toProperties();
    final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
            .createStreamTableSource(propertiesMap);
    assertNotNull(actualSource);
    TableSourceValidation.validateTableSource(actualSource, tableSchema);
}

Example #17

Source File: FlinkTableITCase.java From flink-connectors with Apache License 2.0

4 votes

/**
 * Validates the use of Pravega Table Descriptor to generate the source/sink Table factory to
 * write and read from Pravega stream using {@link StreamTableEnvironment}
 * @throws Exception
 */
@Test
public void testStreamingTableUsingDescriptor() throws Exception {

    final String scope = setupUtils.getScope();
    final String streamName = "stream";
    Stream stream = Stream.of(scope, streamName);
    this.setupUtils.createTestStream(stream.getStreamName(), 1);

    StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment().setParallelism(1);
    StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env,
            EnvironmentSettings.newInstance()
                    // watermark is only supported in blink planner
                    .useBlinkPlanner()
                    .inStreamingMode()
                    .build());

    PravegaConfig pravegaConfig = setupUtils.getPravegaConfig();

    Pravega pravega = new Pravega();
    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("category")
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);
    pravega.tableSourceReaderBuilder()
            .withReaderGroupScope(stream.getScope())
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);

    TableSchema tableSchema = TableSchema.builder()
            .field("category", DataTypes.STRING())
            .field("value", DataTypes.INT())
            .build();

    Schema schema = new Schema().schema(tableSchema);

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(
                new Json()
                        .failOnMissingField(false)
            )
            .withSchema(schema)
            .inAppendMode();

    desc.createTemporaryTable("test");

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSink<?> sink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
            .createStreamTableSink(propertiesMap);
    final TableSource<?> source = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
            .createStreamTableSource(propertiesMap);

    Table table = tableEnv.fromDataStream(env.fromCollection(SAMPLES));

    String tablePathSink = tableEnv.getCurrentDatabase() + "." + "PravegaSink";

    ConnectorCatalogTable<?, ?> connectorCatalogSinkTable = ConnectorCatalogTable.sink(sink, false);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog())
            .get()
            .createTable(
            ObjectPath.fromString(tablePathSink),
            connectorCatalogSinkTable, false);

    table.insertInto("PravegaSink");

    ConnectorCatalogTable<?, ?> connectorCatalogSourceTable = ConnectorCatalogTable.source(source, false);
    String tablePathSource = tableEnv.getCurrentDatabase() + "." + "samples";

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tablePathSource),
            connectorCatalogSourceTable, false);
    // select some sample data from the Pravega-backed table, as a view
    Table view = tableEnv.sqlQuery("SELECT * FROM samples WHERE category IN ('A','B')");

    // write the view to a test sink that verifies the data for test purposes
    tableEnv.toAppendStream(view, SampleRecord.class).addSink(new TestSink(SAMPLES));

    // execute the topology
    try {
        env.execute();
        Assert.fail("expected an exception");
    } catch (Exception e) {
        // we expect the job to fail because the test sink throws a deliberate exception.
        Assert.assertTrue(ExceptionUtils.getRootCause(e) instanceof TestCompletionException);
    }
}

Example #18

Source File: FlinkTableITCase.java From flink-connectors with Apache License 2.0

4 votes

/**
 * Validates the use of Pravega Table Descriptor to generate the source/sink Table factory to
 * write and read from Pravega stream using {@link BatchTableEnvironment}
 * @throws Exception
 */
@Test
public void testBatchTableUsingDescriptor() throws Exception {

    final String scope = setupUtils.getScope();
    final String streamName = "stream";
    Stream stream = Stream.of(scope, streamName);
    this.setupUtils.createTestStream(stream.getStreamName(), 1);

    ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
    env.setParallelism(1);
    BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env);

    PravegaConfig pravegaConfig = setupUtils.getPravegaConfig();

    Pravega pravega = new Pravega();
    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("category")
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);
    pravega.tableSourceReaderBuilder()
            .withReaderGroupScope(stream.getScope())
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(new Json().failOnMissingField(false))
            .withSchema(new Schema().
                    field("category", DataTypes.STRING()).
                    field("value", DataTypes.INT()));
    desc.createTemporaryTable("test");

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSink<?> sink = TableFactoryService.find(BatchTableSinkFactory.class, propertiesMap)
            .createBatchTableSink(propertiesMap);
    final TableSource<?> source = TableFactoryService.find(BatchTableSourceFactory.class, propertiesMap)
            .createBatchTableSource(propertiesMap);

    Table table = tableEnv.fromDataSet(env.fromCollection(SAMPLES));

    String tableSinkPath = tableEnv.getCurrentDatabase() + "." + "PravegaSink";

    ConnectorCatalogTable<?, ?> connectorCatalogTableSink = ConnectorCatalogTable.sink(sink, true);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tableSinkPath),
            connectorCatalogTableSink, false);

    table.insertInto("PravegaSink");
    env.execute();

    String tableSourcePath = tableEnv.getCurrentDatabase() + "." + "samples";

    ConnectorCatalogTable<?, ?> connectorCatalogTableSource = ConnectorCatalogTable.source(source, true);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tableSourcePath),
            connectorCatalogTableSource, false);

    // select some sample data from the Pravega-backed table, as a view
    Table view = tableEnv.sqlQuery("SELECT * FROM samples WHERE category IN ('A','B')");

    // convert the view to a dataset and collect the results for comparison purposes
    List<SampleRecord> results = tableEnv.toDataSet(view, SampleRecord.class).collect();
    Assert.assertEquals(new HashSet<>(SAMPLES), new HashSet<>(results));
}

Example #19

Source File: FlinkTableITCase.java From flink-connectors with Apache License 2.0

4 votes

@Test
public void testStreamTableSinkUsingDescriptor() throws Exception {

    // create a Pravega stream for test purposes
    Stream stream = Stream.of(setupUtils.getScope(), "testStreamTableSinkUsingDescriptor");
    this.setupUtils.createTestStream(stream.getStreamName(), 1);

    // create a Flink Table environment
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment().setParallelism(1);
    StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env,
            EnvironmentSettings.newInstance()
                    // watermark is only supported in blink planner
                    .useBlinkPlanner()
                    .inStreamingMode()
                    .build());

    Table table = tableEnv.fromDataStream(env.fromCollection(SAMPLES));

    Pravega pravega = new Pravega();
    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("category")
            .forStream(stream)
            .withPravegaConfig(setupUtils.getPravegaConfig());

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(new Json().failOnMissingField(true))
            .withSchema(new Schema().
                    field("category", DataTypes.STRING())
                    .field("value", DataTypes.INT()))
            .inAppendMode();
    desc.createTemporaryTable("test");

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSink<?> sink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
            .createStreamTableSink(propertiesMap);

    String tablePath = tableEnv.getCurrentDatabase() + "." + "PravegaSink";

    ConnectorCatalogTable<?, ?> connectorCatalogTable = ConnectorCatalogTable.sink(sink, false);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tablePath),
            connectorCatalogTable, false);

    table.insertInto("PravegaSink");
    env.execute();
}

Example #20

Source File: FlinkTableITCase.java From flink-connectors with Apache License 2.0

4 votes

@Test
public void testStreamTableSinkUsingDescriptorWithWatermark() throws Exception {
    // create a Pravega stream for test purposes
    Stream stream = Stream.of(setupUtils.getScope(), "testStreamTableSinkUsingDescriptorWithWatermark");
    this.setupUtils.createTestStream(stream.getStreamName(), 1);

    // create a Flink Table environment
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment().setParallelism(1);
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env,
            EnvironmentSettings.newInstance()
                    // watermark is only supported in blink planner
                    .useBlinkPlanner()
                    .inStreamingMode()
                    .build());
    DataStream<SampleRecordWithTimestamp> dataStream = env.fromCollection(SAMPLES)
            .map(SampleRecordWithTimestamp::new)
            .assignTimestampsAndWatermarks(new AscendingTimestampExtractor<SampleRecordWithTimestamp>() {
                @Override
                public long extractAscendingTimestamp(SampleRecordWithTimestamp sampleRecordWithTimestamp) {
                    return sampleRecordWithTimestamp.getTimestamp();
                }
            });

    Table table = tableEnv.fromDataStream(dataStream, "category, value, UserActionTime.rowtime");

    Pravega pravega = new Pravega();
    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("category")
            .enableWatermark(true)
            .forStream(stream)
            .withPravegaConfig(setupUtils.getPravegaConfig());

    TableSchema tableSchema = TableSchema.builder()
            .field("category", DataTypes.STRING())
            .field("value", DataTypes.INT())
            .field("timestamp", DataTypes.TIMESTAMP(3))
            .build();

    Schema schema = new Schema().schema(tableSchema);

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(new Json().failOnMissingField(true))
            .withSchema(schema)
            .inAppendMode();
    desc.createTemporaryTable("test");

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSink<?> sink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
            .createStreamTableSink(propertiesMap);

    String tablePath = tableEnv.getCurrentDatabase() + "." + "PravegaSink";

    ConnectorCatalogTable<?, ?> connectorCatalogTable = ConnectorCatalogTable.sink(sink, false);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tablePath),
            connectorCatalogTable, false);

    table.insertInto(tablePath);
    env.execute();
}

Example #21

Source File: FlinkPravegaTableSinkTest.java From flink-connectors with Apache License 2.0

4 votes

@Test
@SuppressWarnings("unchecked")
public void testTableSinkDescriptor() {
    final String cityName = "fruitName";
    final String total = "count";
    final String eventTime = "eventTime";
    final String procTime = "procTime";
    final String controllerUri = "tcp://localhost:9090";
    final long delay = 3000L;
    final String streamName = "test";
    final String scopeName = "test";

    Stream stream = Stream.of(scopeName, streamName);
    PravegaConfig pravegaConfig = PravegaConfig.fromDefaults()
            .withControllerURI(URI.create(controllerUri))
            .withDefaultScope(scopeName);

    // construct table sink using descriptors and table sink factory
    Pravega pravega = new Pravega();
    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField(cityName)
            .withWriterMode(PravegaWriterMode.EXACTLY_ONCE)
            .enableWatermark(true)
            .forStream(stream)
            .enableMetrics(true)
            .withPravegaConfig(pravegaConfig);

    final FlinkPravegaTableSourceTest.TestTableDescriptor testDesc = new FlinkPravegaTableSourceTest.TestTableDescriptor(pravega)
            .withFormat(new Json().failOnMissingField(false))
            .withSchema(
                    new Schema()
                            .field(cityName, DataTypes.STRING())
                            .field(total, DataTypes.BIGINT())
                            .field(eventTime, DataTypes.TIMESTAMP(3))
                            .rowtime(new Rowtime()
                                    .timestampsFromField(eventTime)
                                    .watermarksFromStrategy(new BoundedOutOfOrderTimestamps(delay))
                            )
                            .field(procTime, DataTypes.TIMESTAMP(3)).proctime()
            )
            .inAppendMode();

    final Map<String, String> propertiesMap = testDesc.toProperties();
    final TableSink<?> sink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
            .createStreamTableSink(propertiesMap);

    assertNotNull(sink);
}

Example #22

Source File: ElasticsearchUpsertTableSinkFactoryTestBase.java From flink with Apache License 2.0

4 votes

@Test
public void testTableSink() {
	// prepare parameters for Elasticsearch table sink

	final TableSchema schema = createTestSchema();

	final ElasticsearchUpsertTableSinkBase expectedSink = getExpectedTableSink(
		false,
		schema,
		Collections.singletonList(new Host(HOSTNAME, PORT, SCHEMA)),
		INDEX,
		DOC_TYPE,
		KEY_DELIMITER,
		KEY_NULL_LITERAL,
		new JsonRowSerializationSchema(schema.toRowType()),
		XContentType.JSON,
		new DummyFailureHandler(),
		createTestSinkOptions());

	// construct table sink using descriptors and table sink factory

	final TestTableDescriptor testDesc = new TestTableDescriptor(
			new Elasticsearch()
				.version(getElasticsearchVersion())
				.host(HOSTNAME, PORT, SCHEMA)
				.index(INDEX)
				.documentType(DOC_TYPE)
				.keyDelimiter(KEY_DELIMITER)
				.keyNullLiteral(KEY_NULL_LITERAL)
				.bulkFlushBackoffExponential()
				.bulkFlushBackoffDelay(123L)
				.bulkFlushBackoffMaxRetries(3)
				.bulkFlushInterval(100L)
				.bulkFlushMaxActions(1000)
				.bulkFlushMaxSize("1 MB")
				.failureHandlerCustom(DummyFailureHandler.class)
				.connectionMaxRetryTimeout(100)
				.connectionPathPrefix("/myapp"))
		.withFormat(
			new Json()
				.deriveSchema())
		.withSchema(
			new Schema()
				.field(FIELD_KEY, Types.LONG())
				.field(FIELD_FRUIT_NAME, Types.STRING())
				.field(FIELD_COUNT, Types.DECIMAL())
				.field(FIELD_TS, Types.SQL_TIMESTAMP()))
		.inUpsertMode();

	final Map<String, String> propertiesMap = testDesc.toProperties();
	final TableSink<?> actualSink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
		.createStreamTableSink(propertiesMap);

	assertEquals(expectedSink, actualSink);
}

Example #23

Source File: ElasticsearchUpsertTableSinkFactoryTestBase.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testTableSink() {
	// prepare parameters for Elasticsearch table sink

	final TableSchema schema = createTestSchema();

	final ElasticsearchUpsertTableSinkBase expectedSink = getExpectedTableSink(
		false,
		schema,
		Collections.singletonList(new Host(HOSTNAME, PORT, SCHEMA)),
		INDEX,
		DOC_TYPE,
		KEY_DELIMITER,
		KEY_NULL_LITERAL,
		new JsonRowSerializationSchema(schema.toRowType()),
		XContentType.JSON,
		new DummyFailureHandler(),
		createTestSinkOptions());

	// construct table sink using descriptors and table sink factory

	final TestTableDescriptor testDesc = new TestTableDescriptor(
			new Elasticsearch()
				.version(getElasticsearchVersion())
				.host(HOSTNAME, PORT, SCHEMA)
				.index(INDEX)
				.documentType(DOC_TYPE)
				.keyDelimiter(KEY_DELIMITER)
				.keyNullLiteral(KEY_NULL_LITERAL)
				.bulkFlushBackoffExponential()
				.bulkFlushBackoffDelay(123L)
				.bulkFlushBackoffMaxRetries(3)
				.bulkFlushInterval(100L)
				.bulkFlushMaxActions(1000)
				.bulkFlushMaxSize("1 MB")
				.failureHandlerCustom(DummyFailureHandler.class)
				.connectionMaxRetryTimeout(100)
				.connectionPathPrefix("/myapp"))
		.withFormat(
			new Json()
				.deriveSchema())
		.withSchema(
			new Schema()
				.field(FIELD_KEY, Types.LONG())
				.field(FIELD_FRUIT_NAME, Types.STRING())
				.field(FIELD_COUNT, Types.DECIMAL())
				.field(FIELD_TS, Types.SQL_TIMESTAMP()))
		.inUpsertMode();

	final Map<String, String> propertiesMap = testDesc.toProperties();
	final TableSink<?> actualSink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
		.createStreamTableSink(propertiesMap);

	assertEquals(expectedSink, actualSink);
}