com.hazelcast.function.FunctionEx Java Examples

The following examples show how to use com.hazelcast.function.FunctionEx. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PulsarConsumerBuilder.java    From hazelcast-jet-contrib with Apache License 2.0 6 votes vote down vote up
private ConsumerContext(
        @Nonnull ILogger logger,
        @Nonnull PulsarClient client,
        @Nonnull List<String> topics,
        @Nonnull Map<String, Object> consumerConfig,
        @Nonnull SupplierEx<Schema<M>> schemaSupplier,
        @Nonnull SupplierEx<BatchReceivePolicy> batchReceivePolicySupplier,
        @Nonnull FunctionEx<Message<M>, T> projectionFn
) throws PulsarClientException {

    this.logger = logger;
    this.projectionFn = projectionFn;
    this.client = client;
    this.consumer = client.newConsumer(schemaSupplier.get())
                          .topics(topics)
                          .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
                          .loadConf(consumerConfig)
                          .batchReceivePolicy(batchReceivePolicySupplier.get())
                          .subscriptionType(SubscriptionType.Shared)
                          .subscribe();
}
 
Example #2
Source File: MongoDBSourceBuilder.java    From hazelcast-jet-contrib with Apache License 2.0 6 votes vote down vote up
private static <T, U> SupplierEx<StreamContext<T, U>> contextFn(
        SupplierEx<? extends MongoClient> connectionSupplier,
        FunctionEx<? super MongoClient, ? extends MongoDatabase> databaseFn,
        FunctionEx<? super MongoDatabase, ? extends MongoCollection<? extends T>> collectionFn,
        ConsumerEx<? super MongoClient> destroyFn,
        FunctionEx<? super MongoCollection<? extends T>, ? extends ChangeStreamIterable<? extends T>> searchFn,
        FunctionEx<? super ChangeStreamDocument<? extends T>, U> mapFn,
        FunctionEx<? super MongoClient, ? extends BsonTimestamp> startAtOperationTimeFn

) {
    return () -> {
        MongoClient client = connectionSupplier.get();
        MongoDatabase database = databaseFn.apply(client);
        MongoCollection<? extends T> collection = collectionFn.apply(database);
        ChangeStreamIterable<? extends T> changeStreamIterable = searchFn.apply(collection);
        return new StreamContext<>(client, changeStreamIterable, mapFn, destroyFn, startAtOperationTimeFn);
    };
}
 
Example #3
Source File: PulsarSinkBuilder.java    From hazelcast-jet-contrib with Apache License 2.0 6 votes vote down vote up
private PulsarSinkContext(
        @Nonnull ILogger logger,
        @Nonnull String topic,
        @Nonnull PulsarClient client,
        @Nonnull Map<String, Object> producerConfig,
        @Nonnull SupplierEx<Schema<M>> schemaSupplier,
        @Nonnull FunctionEx<? super E, M> extractValueFn,
        @Nullable FunctionEx<? super E, String> extractKeyFn,
        @Nullable FunctionEx<? super E, Map<String, String>> extractPropertiesFn,
        @Nullable FunctionEx<? super E, Long> extractTimestampFn
) throws PulsarClientException {
    this.logger = logger;
    this.client = client;
    this.producer = client.newProducer(schemaSupplier.get())
                          .topic(topic)
                          .loadConf(producerConfig)
                          .create();
    this.extractKeyFn = extractKeyFn;
    this.extractValueFn = extractValueFn;
    this.extractPropertiesFn = extractPropertiesFn;
    this.extractTimestampFn = extractTimestampFn;
}
 
Example #4
Source File: PulsarSinkBuilder.java    From hazelcast-jet-contrib with Apache License 2.0 6 votes vote down vote up
/**
 * Required fields of Pulsar sink
 *
 * @param topic              Pulsar topic name to publish to
 * @param connectionSupplier Pulsar client supplier
 * @param extractValueFn     extracts the message value from the emitted items.
 * @param schemaSupplier     Pulsar messaging schema supplier.
 */
public PulsarSinkBuilder(
        @Nonnull String topic,
        @Nonnull SupplierEx<PulsarClient> connectionSupplier,
        @Nonnull SupplierEx<Schema<M>> schemaSupplier,
        @Nonnull FunctionEx<? super E, M> extractValueFn
) {
    checkSerializable(connectionSupplier, "connectionSupplier");
    checkSerializable(schemaSupplier, "schemaSupplier");
    checkSerializable(extractValueFn, "extractValueFn");

    this.topic = topic;
    this.producerConfig = getDefaultProducerConfig();
    this.connectionSupplier = connectionSupplier;
    this.schemaSupplier = schemaSupplier;
    this.extractValueFn = extractValueFn;
}
 
Example #5
Source File: MongoDBSourceBuilder.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
BatchContext(
        MongoClient client,
        MongoCollection<? extends T> collection,
        FunctionEx<? super MongoCollection<? extends T>, ? extends FindIterable<? extends T>> searchFn,
        FunctionEx<? super T, U> mapFn,
        ConsumerEx<? super MongoClient> destroyFn
) {
    this.client = client;
    this.mapFn = mapFn;
    this.destroyFn = destroyFn;

    cursor = searchFn.apply(collection).iterator();
}
 
Example #6
Source File: MongoDBSourceBuilder.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
/**
 * Creates and returns the MongoDB {@link StreamSource} which watches
 * the given collection.
 */
@Nonnull
public StreamSource<U> build() {
    checkNotNull(connectionSupplier, "connectionSupplier must be set");
    checkNotNull(databaseFn, "databaseFn must be set");
    checkNotNull(collectionFn, "collectionFn must be set");
    checkNotNull(searchFn, "searchFn must be set");
    checkNotNull(mapFn, "mapFn must be set");

    SupplierEx<StreamContext<T, U>> contextFn = contextFn(connectionSupplier, databaseFn,
            (FunctionEx<? super MongoDatabase, ? extends MongoCollection<? extends T>>) collectionFn, destroyFn,
            searchFn, mapFn, startAtOperationTimeFn);

    return build(contextFn);
}
 
Example #7
Source File: MongoDBSourceBuilder.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
/**
 * Creates and returns the MongoDB {@link BatchSource}.
 */
@Nonnull
public BatchSource<U> build() {
    checkNotNull(connectionSupplier, "connectionSupplier must be set");
    checkNotNull(databaseFn, "databaseFn must be set");
    checkNotNull(collectionFn, "collectionFn must be set");
    checkNotNull(searchFn, "searchFn must be set");
    checkNotNull(mapFn, "mapFn must be set");

    SupplierEx<? extends MongoClient> localConnectionSupplier = connectionSupplier;
    FunctionEx<? super MongoClient, ? extends MongoDatabase> localDatabaseFn = databaseFn;
    FunctionEx<? super MongoDatabase, ? extends MongoCollection<? extends T>> localCollectionFn
            = (FunctionEx<? super MongoDatabase, ? extends MongoCollection<? extends T>>) collectionFn;
    ConsumerEx<? super MongoClient> localDestroyFn = destroyFn;
    FunctionEx<? super MongoCollection<? extends T>, ? extends FindIterable<? extends T>> localSearchFn = searchFn;
    FunctionEx<? super T, U> localMapFn = mapFn;

    return SourceBuilder
            .batch(name, ctx -> {
                MongoClient client = localConnectionSupplier.get();
                MongoCollection<? extends T> collection = localCollectionFn.apply(localDatabaseFn.apply(client));
                return new BatchContext<>(client, collection, localSearchFn, localMapFn, localDestroyFn);
            })
            .<U>fillBufferFn(BatchContext::fillBuffer)
            .destroyFn(BatchContext::close)
            .build();
}
 
Example #8
Source File: MongoDBSourceBuilder.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
private static <T, U> SupplierEx<StreamContext<T, U>> contextFn(
        SupplierEx<? extends MongoClient> connectionSupplier,
        ConsumerEx<? super MongoClient> destroyFn,
        FunctionEx<? super MongoClient, ? extends ChangeStreamIterable<? extends T>> searchFn,
        FunctionEx<? super ChangeStreamDocument<? extends T>, U> mapFn,
        FunctionEx<? super MongoClient, ? extends BsonTimestamp> startAtOperationTimeFn
) {
    return () -> {
        MongoClient client = connectionSupplier.get();
        ChangeStreamIterable<? extends T> changeStreamIterable = searchFn.apply(client);
        return new StreamContext<>(client, changeStreamIterable, mapFn, destroyFn, startAtOperationTimeFn);
    };
}
 
Example #9
Source File: MongoDBSourceBuilder.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
private static <T, U> SupplierEx<StreamContext<T, U>> contextFn(
        SupplierEx<? extends MongoClient> connectionSupplier,
        FunctionEx<? super MongoClient, ? extends MongoDatabase> databaseFn,
        ConsumerEx<? super MongoClient> destroyFn,
        FunctionEx<? super MongoDatabase, ? extends ChangeStreamIterable<? extends T>> searchFn,
        FunctionEx<? super ChangeStreamDocument<? extends T>, U> mapFn,
        FunctionEx<? super MongoClient, ? extends BsonTimestamp> startAtOperationTimeFn
) {
    return () -> {
        MongoClient client = connectionSupplier.get();
        MongoDatabase database = databaseFn.apply(client);
        ChangeStreamIterable<? extends T> changeStreamIterable = searchFn.apply(database);
        return new StreamContext<>(client, changeStreamIterable, mapFn, destroyFn, startAtOperationTimeFn);
    };
}
 
Example #10
Source File: MongoDBSourceBuilder.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
StreamContext(
        MongoClient client,
        ChangeStreamIterable<? extends T> changeStreamIterable,
        FunctionEx<? super ChangeStreamDocument<? extends T>, U> mapFn,
        ConsumerEx<? super MongoClient> destroyFn,
        FunctionEx<? super MongoClient, ? extends BsonTimestamp> startAtOperationTimeFn
) {
    this.client = client;
    this.changeStreamIterable = changeStreamIterable;
    this.mapFn = mapFn;
    this.destroyFn = destroyFn;

    this.timestamp = startAtOperationTimeFn == null ? null : startAtOperationTimeFn.apply(client);
}
 
Example #11
Source File: PulsarConsumerBuilder.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
/**
 * Required fields of Pulsar consumer.
 *
 * @param topics                     the topics to consume, at least one is required
 * @param connectionSupplier         Pulsar client supplier
 * @param schemaSupplier             supplies the schema for consuming messages
 * @param projectionFn               converts a Pulsar message to an emitted item.
 */
public PulsarConsumerBuilder(List<String> topics, SupplierEx<PulsarClient> connectionSupplier,
                             SupplierEx<Schema<M>> schemaSupplier, FunctionEx<Message<M>, T> projectionFn) {
    checkSerializable(topics, "topics");
    checkSerializable(connectionSupplier, "connectionSupplier");
    checkSerializable(schemaSupplier, "schemaSupplier");
    checkSerializable(projectionFn, "projectionFn");

    this.topics = topics;
    this.connectionSupplier = connectionSupplier;
    this.schemaSupplier = schemaSupplier;
    this.projectionFn = projectionFn;
    this.consumerConfig = getDefaultConsumerConfig();
    this.batchReceivePolicySupplier = getDefaultBatchReceivePolicySupplier();
}
 
Example #12
Source File: BreastCancerClassification.java    From hazelcast-jet-demos with Apache License 2.0 5 votes vote down vote up
private static FunctionEx<BatchStage<BreastCancerDiagnostic>, BatchStage<String>> applyPredictionFromModelFile() {
    return stage -> stage.mapUsingService(modelContext(), (modelWrapper, diagnostic) -> {
        BinomialModelPrediction p = modelWrapper.predictBinomial(diagnostic.asRow());
        TumorType predictedTumorType = TumorType.fromString(p.label);
        if (diagnostic.getDiagnosis() == predictedTumorType) {
            return String.format("Match: Actual: %1$-" + 15 + "s", diagnostic.getDiagnosis()) + "Prediction: "
                    + formatPrediction(p, predictedTumorType);
        } else {
            return String.format("Miss: Actual: %1$-" + 15 + "s", diagnostic.getDiagnosis()) + "Prediction: "
                    + formatPrediction(p, predictedTumorType);
        }
    }).setName("Apply H2O classification from loaded MOJO");
}
 
Example #13
Source File: MongoDBSinkBuilder.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
/**
 * Creates and returns the MongoDB {@link Sink} with the components you
 * supplied to this builder.
 */
public Sink<T> build() {
    checkNotNull(connectionSupplier, "connectionSupplier must be set");
    checkNotNull(databaseFn, "databaseFn must be set");
    checkNotNull(collectionFn, "collectionFn must be set");

    SupplierEx<MongoClient> localConnectionSupplier = connectionSupplier;
    FunctionEx<MongoClient, MongoDatabase> localDatabaseFn = databaseFn;
    FunctionEx<MongoDatabase, MongoCollection<T>> localCollectionFn = collectionFn;
    ConsumerEx<MongoClient> localDestroyFn = destroyFn;

    boolean localOrdered = ordered;
    boolean localBypassValidation = bypassValidation;

    return SinkBuilder
            .sinkBuilder(name, ctx -> {
                MongoClient client = localConnectionSupplier.get();
                MongoCollection<T> collection = localCollectionFn.apply(localDatabaseFn.apply(client));
                return new MongoSinkContext<>(client, collection, localDestroyFn, localOrdered, localBypassValidation);
            })
            .<T>receiveFn(MongoSinkContext::addDocument)
            .flushFn(MongoSinkContext::flush)
            .destroyFn(MongoSinkContext::close)
            .preferredLocalParallelism(preferredLocalParallelism)
            .build();

}
 
Example #14
Source File: RedisSources.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
ProcSupplier(
        RedisURI uri,
        Map<K, String> streamOffsets,
        EventTimePolicy<? super T> eventTimePolicy,
        SupplierEx<RedisCodec<K, V>> codecFn,
        FunctionEx<? super StreamMessage<K, V>, ? extends T> projectionFn
) {
    this.uri = uri;
    this.streamOffsets = streamOffsets;
    this.eventTimePolicy = eventTimePolicy;
    this.codecFn = codecFn;
    this.projectionFn = projectionFn;
}
 
Example #15
Source File: RedisSources.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
MetaSupplier(
        RedisURI uri,
        Map<K, String> streamOffsets,
        EventTimePolicy<? super T> eventTimePolicy,
        SupplierEx<RedisCodec<K, V>> codecFn,
        FunctionEx<? super StreamMessage<K, V>, ? extends T> projectionFn
) {
    this.uri = uri;
    this.streamOffsets = streamOffsets;
    this.eventTimePolicy = eventTimePolicy;
    this.codecFn = codecFn;
    this.projectionFn = projectionFn;
}
 
Example #16
Source File: RedisSources.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
static <K, V, T> ProcessorMetaSupplier streamRedisP(
        RedisURI uri,
        Map<K, String> streamOffsets,
        EventTimePolicy<? super T> eventTimePolicy,
        SupplierEx<RedisCodec<K, V>> codecFn,
        FunctionEx<? super StreamMessage<K, V>, ? extends T> projectionFn
) {
    return new MetaSupplier<>(uri, streamOffsets, eventTimePolicy, codecFn, projectionFn);
}
 
Example #17
Source File: RedisSources.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
StreamRedisP(
        RedisURI uri,
        Map<K, String> streamOffsets,
        EventTimePolicy<? super T> eventTimePolicy,
        SupplierEx<RedisCodec<K, V>> codecFn,
        FunctionEx<? super StreamMessage<K, V>, ? extends T> projectionFn
) {

    this.uri = uri;
    this.streamOffsets = streamOffsets;
    this.codecFn = codecFn;
    this.projectionFn = projectionFn;

    this.eventTimeMapper = new EventTimeMapper<>(eventTimePolicy);
}
 
Example #18
Source File: RedisSources.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
HashContext(
        RedisURI uri,
        K hash,
        SupplierEx<RedisCodec<K, V>> codecSupplier,
        FunctionEx<Map.Entry<K, V>, T> mapFn
) {
    this.client = RedisClient.create(uri);
    this.connection = client.connect(codecSupplier.get());

    this.mapFn = mapFn;

    RedisAsyncCommands<K, V> commands = connection.async();
    this.commandFuture = commands.hgetall(this, hash);
}
 
Example #19
Source File: RedisSinks.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
private StreamContext(
        RedisURI uri,
        K stream,
        SupplierEx<RedisCodec<K, V>> codecFn,
        FunctionEx<T, Map<K, V>> mapFn
) {
    this.stream = stream;
    this.mapFn = mapFn;

    redisClient = RedisClient.create(uri);
    connection = redisClient.connect(codecFn.get());
}
 
Example #20
Source File: PulsarTestSupport.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
protected static StreamSource<String> setupReaderSource(String topicName,
                                                        FunctionEx<Message<byte[]>, String> projectionFn) {
    return PulsarSources.pulsarReader(
            topicName,
            () -> PulsarClient.builder().serviceUrl(getServiceUrl()).build(),
            () -> Schema.BYTES,
            projectionFn);
}
 
Example #21
Source File: PulsarTestSupport.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
protected static StreamSource<String> setupConsumerSource(String topicName,
                                                          FunctionEx<Message<byte[]>, String> projectionFn) {
    return PulsarSources.pulsarConsumer(
            topicName,
            () -> PulsarClient.builder().serviceUrl(getServiceUrl()).build(),
            () -> Schema.BYTES,
            projectionFn);
}
 
Example #22
Source File: RedisSinks.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
SortedSetContext(RedisURI uri, K sortedSet, RedisCodec<K, V> codec, FunctionEx<T, Double> scoreFn,
                 FunctionEx<T, V> valueFn) {
    this.client = RedisClient.create(uri);
    this.connection = client.connect(codec);
    this.commands = connection.async();
    this.sortedSet = sortedSet;
    this.scoreFn = scoreFn;
    this.valueFn = valueFn;
}
 
Example #23
Source File: PulsarReaderBuilder.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
private ReaderContext(
        @Nonnull ILogger logger,
        @Nonnull PulsarClient client,
        @Nonnull String topic,
        @Nonnull Map<String, Object> readerConfig,
        @Nonnull SupplierEx<Schema<M>> schemaSupplier,
        @Nonnull FunctionEx<Message<M>, T> projectionFn
) {
    this.logger = logger;
    this.client = client;
    this.topic = topic;
    this.readerConfig = readerConfig;
    this.schema = schemaSupplier.get();
    this.projectionFn = projectionFn;
}
 
Example #24
Source File: PulsarReaderBuilder.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
/**
 * Required fields of Pulsar reader
 *
 * @param topic              Pulsar topic name to consume from
 * @param connectionSupplier Pulsar client supplier
 * @param schemaSupplier     Pulsar messaging schema supplier.
 * @param projectionFn       converts a Pulsar message to an emitted item.
 */
public PulsarReaderBuilder(
        @Nonnull String topic,
        @Nonnull SupplierEx<PulsarClient> connectionSupplier,
        @Nonnull SupplierEx<Schema<M>> schemaSupplier,
        @Nonnull FunctionEx<Message<M>, T> projectionFn) {
    checkSerializable(connectionSupplier, "connectionSupplier");
    checkSerializable(schemaSupplier, "schemaSupplier");
    checkSerializable(projectionFn, "projectionFn");
    this.topic = topic;
    this.connectionSupplier = connectionSupplier;
    this.readerConfig = getDefaultReaderConfig();
    this.schemaSupplier = schemaSupplier;
    this.projectionFn = projectionFn;
}
 
Example #25
Source File: RedisSources.java    From hazelcast-jet-contrib with Apache License 2.0 4 votes vote down vote up
/**
 * Creates a {@link StreamSource} which reads all items from multiple Redis
 * Streams (starting from given indexes) and emits them as they arrive.
 * <p>
 * Here is an example which reads all elements from two different Redis
 * Streams, maps the objects received to a stream specific ID and drains
 * the results out to some generic sink.
 * <pre>{@code
 * Map<String, String> streamOffsets = new HashMap<>();
 * streamOffsets.put("streamA", "0");
 * streamOffsets.put("streamB", "0");
 *
 * RedisURI uri = RedisURI.create("redis://localhost/");
 *
 * Pipeline.create()
 *     .readFrom(RedisSources.stream("source", uri, streamOffsets,
 *                      StringCodec::new,
 *                      mes -> mes.getStream() + " - " + mes.getId()))
 *     .withoutTimestamps()
 *     .writeTo(sink);
 * }</pre>
 *
 * @param <K>           type of the stream identifier and of fields (keys of
 *                      the Redis Stream entry's body map)
 * @param <V>           type of the field values in the message body
 * @param <T>           type of data coming out of the stream, the result of
 *                      applying the projection function over {@link
 *                      StreamMessage} instances
 * @param name          name of the source being created
 * @param uri           URI of the Redis server
 * @param streamOffsets map keyed by stream identifiers, containing offset
 *                      values from which to start element retrieval of each
 *                      stream
 * @param codecFn       supplier of {@link RedisCodec} instances, used in
 *                      turn for serializing/deserializing keys and values
 * @param projectionFn  built in mapping function of the source which can be
 *                      used to map {@link StreamMessage} instances received
 *                      from Redis to an arbitrary type of output; this
 *                      could be done by an external mapping function in the
 *                      pipeline too, but it's included for convenience
 * @return source to use in {@link com.hazelcast.jet.pipeline.Pipeline#readFrom}
 */
@Nonnull
public static <K, V, T> StreamSource<T> stream(
        @Nonnull String name,
        @Nonnull RedisURI uri,
        @Nonnull Map<K, String> streamOffsets,
        @Nonnull SupplierEx<RedisCodec<K, V>> codecFn,
        @Nonnull FunctionEx<? super StreamMessage<K, V>, ? extends T> projectionFn
) {
    Objects.requireNonNull(name, "name");
    Objects.requireNonNull(uri, "uri");
    Objects.requireNonNull(streamOffsets, "streamOffsets");
    Objects.requireNonNull(codecFn, "codecFn");
    Objects.requireNonNull(projectionFn, "projectionFn");

    Util.checkSerializable(codecFn, "codecFn");
    Util.checkSerializable(projectionFn, "projectionFn");

    return streamFromProcessorWithWatermarks(name, false,
            w -> StreamRedisP.streamRedisP(uri, streamOffsets, w, codecFn, projectionFn));
}
 
Example #26
Source File: RedisSinks.java    From hazelcast-jet-contrib with Apache License 2.0 4 votes vote down vote up
HashContext(RedisURI uri, K hash, RedisCodec<K, V> codec, FunctionEx<T, K> keyFn,
            FunctionEx<T, V> valueFn) {
    this.client = RedisClient.create(uri);
    this.connection = client.connect(codec);
    this.commands = connection.async();
    this.hash = hash;
    this.keyFn = keyFn;
    this.valueFn = valueFn;
}
 
Example #27
Source File: PulsarSinks.java    From hazelcast-jet-contrib with Apache License 2.0 3 votes vote down vote up
/**
 * Convenience for {@link #builder(String, SupplierEx, SupplierEx, FunctionEx)}.
 * It creates a basic Pulsar sink that connect the topic.
 *
 * @param topic              Pulsar topic name to publish to
 * @param connectionSupplier Pulsar client supplier
 * @param schemaSupplier     extracts the message value from the emitted items.
 * @param extractValueFn     Pulsar messaging schema supplier.
 * @param <E>                the type of stream items that sink accepts
 * @param <M>                the type of the message published by {@code PulsarProducer}
 */
public static <E, M> Sink<E> pulsarSink(
        @Nonnull String topic,
        @Nonnull SupplierEx<PulsarClient> connectionSupplier,
        @Nonnull SupplierEx<Schema<M>> schemaSupplier,
        @Nonnull FunctionEx<? super E, M> extractValueFn
) {
    return PulsarSinks.<E, M>builder(topic, connectionSupplier, schemaSupplier, extractValueFn).build();
}
 
Example #28
Source File: PulsarSources.java    From hazelcast-jet-contrib with Apache License 2.0 3 votes vote down vote up
/**
 * Returns a builder object that offers a step-by-step fluent API to build
 * a custom Pulsar consumer {@link StreamSource} for the Pipeline API.
 * <p>
 * Pulsar consumer source is a distributed, timestamped {@link StreamSource}
 * which reads messages from Pulsar topics for data ingestion to Jet
 * pipelines. This source does not have fault-tolerance support. It uses
 * the Consumer API of the Pulsar client. It can be used to subscribe
 * partitioned topics. It uses higher level abstraction of Pulsar that
 * is called "shared subscription" that allows multiple consumers to consume
 * from the topics at the same time. The messages are sent round-robin to
 * each connected consumer. Broker determines which consumer will receive a
 * message from which topic partition. It does not require one-to-one
 * mapping between partitions and consumers. Multiple consumers can get
 * messages from same partition. With this source, the message ordering is
 * not preserved.
 * <p>
 * Example usage:
 * <pre>{@code
 *
 * StreamSource<String> pulsarSource = PulsarSources.pulsarConsumerBuilder(
 *          Arrays.asList(topicName, topicName2 ...),
 *          () -> PulsarClient.builder()
 *                            .serviceUrl("pulsar://exampleserviceurl")
 *                            .build(), // Client Supplier
 *          () -> Schema.BYTES, // Schema Supplier Function
 *          x -> new String(x.getData(), StandardCharsets.UTF_8)
 *                                       // Projection function that converts
 *                                       // receiving bytes to String
 *                                       // before emitting.
 *          ).build();
 *
 *  Pipeline pipeline = Pipeline.create();
 *  StreamStage<Status> srcStage = p.readFrom(pulsarSource);
 *
 *  }</pre>
 *
 * @param topics             the topics to consume, at least one is required
 * @param connectionSupplier Pulsar client supplier
 * @param schemaSupplier     supplies the schema for consuming messages
 * @param projectionFn       converts a Pulsar message to an emitted item.
 * @param <M>                the type of the message read by {@code PulsarConsumer}
 * @param <T>                the type of data emitted from {@code StreamSource}
 * @return {@link PulsarConsumerBuilder} that used to create a {@link StreamSource}
 */

public static <M, T> PulsarConsumerBuilder<M, T> pulsarConsumerBuilder(
        @Nonnull List<String> topics,
        @Nonnull SupplierEx<PulsarClient> connectionSupplier,
        @Nonnull SupplierEx<Schema<M>> schemaSupplier,
        @Nonnull FunctionEx<Message<M>, T> projectionFn

) {
    return new PulsarConsumerBuilder<>(topics, connectionSupplier, schemaSupplier, projectionFn);
}
 
Example #29
Source File: PulsarSources.java    From hazelcast-jet-contrib with Apache License 2.0 3 votes vote down vote up
/**
 * See the {@link #pulsarConsumerBuilder(List, SupplierEx, SupplierEx, FunctionEx)}
 * It gets a single String for the topic name in case it should read only
 * from a single topic.
 *
 * @param topic the single topic to consume
 * @param connectionSupplier Pulsar client supplier
 * @param schemaSupplier     supplies the schema for consuming messages
 * @param projectionFn       converts a Pulsar message to an emitted item.
 * @param <M>                the type of the message read by {@code PulsarConsumer}
 * @param <T>                the type of data emitted from {@code StreamSource}
 * @return {@link PulsarConsumerBuilder} that used to create a {@link StreamSource}
 */
public static <M, T> PulsarConsumerBuilder<M, T> pulsarConsumerBuilder(
        @Nonnull String topic,
        @Nonnull SupplierEx<PulsarClient> connectionSupplier,
        @Nonnull SupplierEx<Schema<M>> schemaSupplier,
        @Nonnull FunctionEx<Message<M>, T> projectionFn

) {
    return pulsarConsumerBuilder(Collections.singletonList(topic), connectionSupplier, schemaSupplier, projectionFn);
}
 
Example #30
Source File: PulsarSources.java    From hazelcast-jet-contrib with Apache License 2.0 3 votes vote down vote up
/**
 * Convenience for {@link #pulsarConsumerBuilder(String, SupplierEx, SupplierEx, FunctionEx)}.
 * It creates a basic Pulsar consumer that connects the topic by using Pulsar client.
 * <p>
 *
 * @param topic              the single topic to consume
 * @param connectionSupplier Pulsar client supplier
 * @param schemaSupplier     supplies the schema for consuming messages
 * @param projectionFn       converts a Pulsar message to an emitted item.
 * @param <M>                the type of the message read by {@code pulsarReader}
 * @param <T>                the type of data emitted from {@code StreamSource}
 * @return {@link StreamSource}
 */
public static <M, T> StreamSource<T> pulsarConsumer(
        @Nonnull String topic,
        @Nonnull SupplierEx<PulsarClient> connectionSupplier,
        @Nonnull SupplierEx<Schema<M>> schemaSupplier,
        @Nonnull FunctionEx<Message<M>, T> projectionFn
) {
    return PulsarSources.pulsarConsumerBuilder(topic, connectionSupplier, schemaSupplier, projectionFn)
                        .build();
}