org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase Java Examples

The following examples show how to use org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Elasticsearch6UpsertTableSinkFactoryTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
protected ElasticsearchUpsertTableSinkBase getExpectedTableSink(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions) {
	return new Elasticsearch6UpsertTableSink(
		isAppendOnly,
		schema,
		hosts,
		index,
		docType,
		keyDelimiter,
		keyNullLiteral,
		serializationSchema,
		contentType,
		failureHandler,
		sinkOptions);
}
Example #2
Source File: Elasticsearch6UpsertTableSinkFactoryTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
protected ElasticsearchUpsertTableSinkBase getExpectedTableSink(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions) {
	return new Elasticsearch6UpsertTableSink(
		isAppendOnly,
		schema,
		hosts,
		index,
		docType,
		keyDelimiter,
		keyNullLiteral,
		serializationSchema,
		contentType,
		failureHandler,
		sinkOptions);
}
Example #3
Source File: Elasticsearch7UpsertTableSinkFactory.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
protected ElasticsearchUpsertTableSinkBase createElasticsearchUpsertTableSink(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions) {

	return new Elasticsearch7UpsertTableSink(
		isAppendOnly,
		schema,
		hosts,
		index,
		keyDelimiter,
		keyNullLiteral,
		serializationSchema,
		contentType,
		failureHandler,
		sinkOptions);
}
Example #4
Source File: Elasticsearch7UpsertTableSinkFactoryTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
protected ElasticsearchUpsertTableSinkBase getExpectedTableSink(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions,
		IndexGenerator indexGenerator) {
	return new Elasticsearch7UpsertTableSink(
		isAppendOnly,
		schema,
		hosts,
		index,
		keyDelimiter,
		keyNullLiteral,
		serializationSchema,
		contentType,
		failureHandler,
		sinkOptions);
}
Example #5
Source File: Elasticsearch6UpsertTableSink.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
protected ElasticsearchUpsertTableSinkBase copy(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions,
		RequestFactory requestFactory) {

	return new Elasticsearch6UpsertTableSink(
		isAppendOnly,
		schema,
		hosts,
		index,
		docType,
		keyDelimiter,
		keyNullLiteral,
		serializationSchema,
		contentType,
		failureHandler,
		sinkOptions);
}
Example #6
Source File: Elasticsearch6UpsertTableSinkFactory.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
protected ElasticsearchUpsertTableSinkBase createElasticsearchUpsertTableSink(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions) {

	return new Elasticsearch6UpsertTableSink(
		isAppendOnly,
		schema,
		hosts,
		index,
		docType,
		keyDelimiter,
		keyNullLiteral,
		serializationSchema,
		contentType,
		failureHandler,
		sinkOptions);
}
Example #7
Source File: Elasticsearch6UpsertTableSink.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected ElasticsearchUpsertTableSinkBase copy(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions,
		RequestFactory requestFactory) {

	return new Elasticsearch6UpsertTableSink(
		isAppendOnly,
		schema,
		hosts,
		index,
		docType,
		keyDelimiter,
		keyNullLiteral,
		serializationSchema,
		contentType,
		failureHandler,
		sinkOptions);
}
Example #8
Source File: Elasticsearch6UpsertTableSinkFactory.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected ElasticsearchUpsertTableSinkBase createElasticsearchUpsertTableSink(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions) {

	return new Elasticsearch6UpsertTableSink(
		isAppendOnly,
		schema,
		hosts,
		index,
		docType,
		keyDelimiter,
		keyNullLiteral,
		serializationSchema,
		contentType,
		failureHandler,
		sinkOptions);
}
Example #9
Source File: Elasticsearch6SinkFunction.java    From alchemy with Apache License 2.0 5 votes vote down vote up 
public Elasticsearch6SinkFunction(String index, Integer fieldIndex, String docType, String keyDelimiter, String keyNullLiteral, SerializationSchema<Row> serializationSchema, XContentType contentType, ElasticsearchUpsertTableSinkBase.RequestFactory requestFactory, int[] keyFieldIndices) {
    this.index = index;
    this.fieldIndex = fieldIndex;
    this.docType = Preconditions.checkNotNull(docType);
    this.keyDelimiter = Preconditions.checkNotNull(keyDelimiter);
    this.serializationSchema = Preconditions.checkNotNull(serializationSchema);
    this.contentType = Preconditions.checkNotNull(contentType);
    this.keyFieldIndices = Preconditions.checkNotNull(keyFieldIndices);
    this.requestFactory = Preconditions.checkNotNull(requestFactory);
    this.keyNullLiteral = Preconditions.checkNotNull(keyNullLiteral);
}
Example #10
Source File: Elasticsearch6TableSink.java    From alchemy with Apache License 2.0 5 votes vote down vote up 
private List<ElasticsearchUpsertTableSinkBase.Host> createHosts(List<String> hosts) {
    Preconditions.checkArgument(hosts != null && !hosts.isEmpty());
    List<ElasticsearchUpsertTableSinkBase.Host> esHosts = new ArrayList<>(hosts.size());
    for (String host : hosts) {
        String[] array = host.split(":");
        esHosts.add(new ElasticsearchUpsertTableSinkBase.Host(array[0], Integer.valueOf(array[1]), "http"));
    }
    return esHosts;
}
Example #11
Source File: Elasticsearch7UpsertTableSink.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected ElasticsearchUpsertTableSinkBase copy(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions,
		RequestFactory requestFactory) {

	return new Elasticsearch7UpsertTableSink(
		isAppendOnly,
		schema,
		hosts,
		index,
		keyDelimiter,
		keyNullLiteral,
		serializationSchema,
		contentType,
		failureHandler,
		sinkOptions);
}
Example #12
Source File: Elasticsearch6UpsertTableSink.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected ElasticsearchUpsertTableSinkBase copy(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions,
		RequestFactory requestFactory) {

	return new Elasticsearch6UpsertTableSink(
		isAppendOnly,
		schema,
		hosts,
		index,
		docType,
		keyDelimiter,
		keyNullLiteral,
		serializationSchema,
		contentType,
		failureHandler,
		sinkOptions);
}
Example #13
Source File: Elasticsearch6UpsertTableSinkFactory.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected ElasticsearchUpsertTableSinkBase createElasticsearchUpsertTableSink(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions) {

	return new Elasticsearch6UpsertTableSink(
		isAppendOnly,
		schema,
		hosts,
		index,
		docType,
		keyDelimiter,
		keyNullLiteral,
		serializationSchema,
		contentType,
		failureHandler,
		sinkOptions);
}
Example #14
Source File: Elasticsearch6UpsertTableSinkFactoryTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected ElasticsearchUpsertTableSinkBase getExpectedTableSink(
		boolean isAppendOnly,
		TableSchema schema,
		List<Host> hosts,
		String index,
		String docType,
		String keyDelimiter,
		String keyNullLiteral,
		SerializationSchema<Row> serializationSchema,
		XContentType contentType,
		ActionRequestFailureHandler failureHandler,
		Map<SinkOption, String> sinkOptions,
		IndexGenerator indexGenerator) {
	return new Elasticsearch6UpsertTableSink(
		isAppendOnly,
		schema,
		hosts,
		index,
		docType,
		keyDelimiter,
		keyNullLiteral,
		serializationSchema,
		contentType,
		failureHandler,
		sinkOptions);
}
Example #15
Source File: Elasticsearch6TableSink.java    From alchemy with Apache License 2.0 4 votes vote down vote up 
private SinkFunction<Tuple2<Boolean, Row>> createSinkFunction(List<ElasticsearchUpsertTableSinkBase.Host> hosts, ActionRequestFailureHandler failureHandler, Elasticsearch6SinkFunction upsertFunction) {
    final List<HttpHost> httpHosts = hosts.stream()
            .map((host) -> new HttpHost(host.hostname, host.port, host.protocol))
            .collect(Collectors.toList());

    final ElasticsearchSink.Builder<Tuple2<Boolean, Row>> builder = new ElasticsearchSink.Builder(httpHosts, upsertFunction);

    builder.setFailureHandler(failureHandler);
    final Map<String, String> bulkConfig = elasticsearch6Properties.getConfig();
    if (bulkConfig != null) {
        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_MAX_ACTIONS))
                .ifPresent(v -> builder.setBulkFlushMaxActions(Integer.valueOf(v)));

        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_MAX_SIZE))
                .ifPresent(v -> builder.setBulkFlushMaxSizeMb(MemorySize.parse(v).getMebiBytes()));

        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_INTERVAL))
                .ifPresent(v -> builder.setBulkFlushInterval(Long.valueOf(v)));

        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_BACKOFF))
                .ifPresent(v -> builder.setBulkFlushBackoff(Boolean.valueOf(v)));

        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_BACKOFF_TYPE))
                .ifPresent(v -> builder.setBulkFlushBackoffType(ElasticsearchSinkBase.FlushBackoffType.valueOf(v)));

        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_BACKOFF_RETRIES))
                .ifPresent(v -> builder.setBulkFlushBackoffRetries(Integer.valueOf(v)));

        Optional.ofNullable(bulkConfig.get(BULK_FLUSH_BACKOFF_DELAY))
                .ifPresent(v -> builder.setBulkFlushBackoffDelay(Long.valueOf(v)));
    }
    builder.setRestClientFactory(
            new DefaultRestClientFactory(
                    elasticsearch6Properties.getMaxRetryTimeoutMills(),
                    elasticsearch6Properties.getPathPrefix()));

    final ElasticsearchSink<Tuple2<Boolean, Row>> sink = builder.build();

    Optional.ofNullable(elasticsearch6Properties.getDisableFlushOnCheckpoint())
            .ifPresent(v -> {
                if (Boolean.valueOf(v)) {
                    sink.disableFlushOnCheckpoint();
                }
            });

    return sink;
}