org.apache.flink.streaming.api.functions.source.ParallelSourceFunction Java Examples

The following examples show how to use org.apache.flink.streaming.api.functions.source.ParallelSourceFunction. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: StreamExecutionEnvironment.java From flink with Apache License 2.0

6 votes

/**
 * Ads a data source with a custom type information thus opening a
 * {@link DataStream}. Only in very special cases does the user need to
 * support type information. Otherwise use
 * {@link #addSource(org.apache.flink.streaming.api.functions.source.SourceFunction)}
 *
 * @param function
 * 		the user defined function
 * @param sourceName
 * 		Name of the data source
 * @param <OUT>
 * 		type of the returned stream
 * @param typeInfo
 * 		the user defined type information for the stream
 * @return the data stream constructed
 */
@SuppressWarnings("unchecked")
public <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function, String sourceName, TypeInformation<OUT> typeInfo) {

	if (function instanceof ResultTypeQueryable) {
		typeInfo = ((ResultTypeQueryable<OUT>) function).getProducedType();
	}
	if (typeInfo == null) {
		try {
			typeInfo = TypeExtractor.createTypeInfo(
					SourceFunction.class,
					function.getClass(), 0, null, null);
		} catch (final InvalidTypesException e) {
			typeInfo = (TypeInformation<OUT>) new MissingTypeInfo(sourceName, e);
		}
	}

	boolean isParallel = function instanceof ParallelSourceFunction;

	clean(function);

	final StreamSource<OUT, ?> sourceOperator = new StreamSource<>(function);
	return new DataStreamSource<>(this, typeInfo, sourceOperator, isParallel, sourceName);
}

Example #2

Source File: StreamExecutionEnvironment.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Ads a data source with a custom type information thus opening a
 * {@link DataStream}. Only in very special cases does the user need to
 * support type information. Otherwise use
 * {@link #addSource(org.apache.flink.streaming.api.functions.source.SourceFunction)}
 *
 * @param function
 * 		the user defined function
 * @param sourceName
 * 		Name of the data source
 * @param <OUT>
 * 		type of the returned stream
 * @param typeInfo
 * 		the user defined type information for the stream
 * @return the data stream constructed
 */
@SuppressWarnings("unchecked")
public <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function, String sourceName, TypeInformation<OUT> typeInfo) {

	if (typeInfo == null) {
		if (function instanceof ResultTypeQueryable) {
			typeInfo = ((ResultTypeQueryable<OUT>) function).getProducedType();
		} else {
			try {
				typeInfo = TypeExtractor.createTypeInfo(
						SourceFunction.class,
						function.getClass(), 0, null, null);
			} catch (final InvalidTypesException e) {
				typeInfo = (TypeInformation<OUT>) new MissingTypeInfo(sourceName, e);
			}
		}
	}

	boolean isParallel = function instanceof ParallelSourceFunction;

	clean(function);
	StreamSource<OUT, ?> sourceOperator;
	if (function instanceof StoppableFunction) {
		sourceOperator = new StoppableStreamSource<>(cast2StoppableSourceFunction(function));
	} else {
		sourceOperator = new StreamSource<>(function);
	}

	return new DataStreamSource<>(this, typeInfo, sourceOperator, isParallel, sourceName);
}

Example #3

Source File: ParrelSourceFunctionDemo.java From blog_demos with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    //并行度为2
    env.setParallelism(2);

    DataStream<Tuple2<Integer,Integer>> dataStream = env.addSource(new ParallelSourceFunction<Tuple2<Integer, Integer>>() {

        private volatile boolean isRunning = true;

        @Override
        public void run(SourceContext<Tuple2<Integer, Integer>> ctx) throws Exception {
            int i = 0;
            while (isRunning) {
                ctx.collect(new Tuple2<>(i++ % 5, 1));
                Thread.sleep(1000);
                if(i>9){
                    break;
                }
            }
        }

        @Override
        public void cancel() {
            isRunning = false;
        }
    });

    dataStream
            .keyBy(0)
            .timeWindow(Time.seconds(2))
            .sum(1)
            .print();

    env.execute("Customize DataSource demo : ParallelSourceFunction");
}

Example #4

Source File: TestableStreamingJobTest.java From flink-training-exercises with Apache License 2.0

5 votes

@Test
public void testCompletePipeline() throws Exception {

	// Arrange
	ParallelSourceFunction<Long> source =
			new ParallelCollectionSource(Arrays.asList(1L, 10L, -10L));
	SinkCollectingLongs sink = new SinkCollectingLongs();
	TestableStreamingJob job = new TestableStreamingJob(source, sink);

	// Act
	job.execute();

	// Assert
	assertThat(sink.result).containsExactlyInAnyOrder(2L, 11L, -9L);
}

Example #5

Source File: StreamingJobGraphGeneratorTest.java From flink with Apache License 2.0

5 votes

private JobGraph createJobGraphForManagedMemoryFractionTest(
	final List<ResourceSpec> resourceSpecs,
	@Nullable final List<Integer> managedMemoryWeights) throws Exception {

	final Method opMethod = getSetResourcesMethodAndSetAccessible(SingleOutputStreamOperator.class);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	final DataStream<Integer> source = env.addSource(new ParallelSourceFunction<Integer>() {
		@Override
		public void run(SourceContext<Integer> ctx) {
		}

		@Override
		public void cancel() {
		}
	});
	opMethod.invoke(source, resourceSpecs.get(0));

	// CHAIN(source -> map1) in default slot sharing group
	final DataStream<Integer> map1 = source.map((MapFunction<Integer, Integer>) value -> value);
	opMethod.invoke(map1, resourceSpecs.get(1));

	// CHAIN(map2) in default slot sharing group
	final DataStream<Integer> map2 = map1.rebalance().map((MapFunction<Integer, Integer>) value -> value);
	opMethod.invoke(map2, resourceSpecs.get(2));

	// CHAIN(map3) in test slot sharing group
	final DataStream<Integer> map3 = map2.rebalance().map(value -> value).slotSharingGroup("test");
	opMethod.invoke(map3, resourceSpecs.get(3));

	if (managedMemoryWeights != null) {
		source.getTransformation().setManagedMemoryWeight(managedMemoryWeights.get(0));
		map1.getTransformation().setManagedMemoryWeight(managedMemoryWeights.get(1));
		map2.getTransformation().setManagedMemoryWeight(managedMemoryWeights.get(2));
		map3.getTransformation().setManagedMemoryWeight(managedMemoryWeights.get(3));
	}

	return StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
}

Example #6

Source File: Sink2ES7Main.java From flink-learning with Apache License 2.0

4 votes

public static void main(String[] args) throws Exception {
    final ParameterTool parameterTool = ExecutionEnvUtil.createParameterTool(args);
    StreamExecutionEnvironment env = ExecutionEnvUtil.prepare(parameterTool);

    List<HttpHost> esAddresses = ESSinkUtil.getEsAddresses(parameterTool.get(ELASTICSEARCH_HOSTS));
    int bulkSize = parameterTool.getInt(ELASTICSEARCH_BULK_FLUSH_MAX_ACTIONS, 40);
    int sinkParallelism = parameterTool.getInt(STREAM_SINK_PARALLELISM, 1);

    log.info("-----esAddresses = {}, parameterTool = {}, ", esAddresses, parameterTool);


    DataStreamSource<MetricEvent> data = env.addSource(new ParallelSourceFunction<MetricEvent>() {
        @Override
        public void run(SourceContext<MetricEvent> context) throws Exception {
            while (true) {
                //just for test

                Map<String, Object> fields = new HashMap<>();
                fields.put("system", 10);
                fields.put("user", 20);
                fields.put("idle", 70);

                Map<String, String> tags = new HashMap<>();
                tags.put("cluster_name", "zhisheng");
                tags.put("host_ip", "11.0.11.0");

                MetricEvent metricEvent = MetricEvent.builder()
                        .name("cpu")
                        .timestamp(System.currentTimeMillis())
                        .fields(fields)
                        .tags(tags)
                        .build();

                context.collect(metricEvent);
                Thread.sleep(200);
            }
        }

        @Override
        public void cancel() {
        }
    });

    ESSinkUtil.addSink(esAddresses, bulkSize, sinkParallelism, data,
            (MetricEvent metric, RuntimeContext runtimeContext, RequestIndexer requestIndexer) -> {
                requestIndexer.add(Requests.indexRequest()
                        .index(ZHISHENG + "_" + metric.getName())
                        .source(GsonUtil.toJSONBytes(metric), XContentType.JSON));
            },
            parameterTool);

    env.execute("flink learning connectors es7");
}

Example #7

Source File: Sink2ES7Main.java From flink-learning with Apache License 2.0

4 votes

public static void main(String[] args) throws Exception {
    final ParameterTool parameterTool = ExecutionEnvUtil.createParameterTool(args);
    StreamExecutionEnvironment env = ExecutionEnvUtil.prepare(parameterTool);

    List<HttpHost> esAddresses = ESSinkUtil.getEsAddresses(parameterTool.get(ELASTICSEARCH_HOSTS));
    int bulkSize = parameterTool.getInt(ELASTICSEARCH_BULK_FLUSH_MAX_ACTIONS, 40);
    int sinkParallelism = parameterTool.getInt(STREAM_SINK_PARALLELISM, 1);

    log.info("-----esAddresses = {}, parameterTool = {}, ", esAddresses, parameterTool);


    DataStreamSource<MetricEvent> data = env.addSource(new ParallelSourceFunction<MetricEvent>() {
        @Override
        public void run(SourceContext<MetricEvent> context) throws Exception {
            while (true) {
                //just for test

                Map<String, Object> fields = new HashMap<>();
                fields.put("system", 10);
                fields.put("user", 20);
                fields.put("idle", 70);

                Map<String, String> tags = new HashMap<>();
                tags.put("cluster_name", "zhisheng");
                tags.put("host_ip", "11.0.11.0");

                MetricEvent metricEvent = MetricEvent.builder()
                        .name("cpu")
                        .timestamp(System.currentTimeMillis())
                        .fields(fields)
                        .tags(tags)
                        .build();

                context.collect(metricEvent);
                Thread.sleep(200);
            }
        }

        @Override
        public void cancel() {
        }
    });

    ESSinkUtil.addSink(esAddresses, bulkSize, sinkParallelism, data,
            (MetricEvent metric, RuntimeContext runtimeContext, RequestIndexer requestIndexer) -> {
                requestIndexer.add(Requests.indexRequest()
                        .index(ZHISHENG + "_" + metric.getName())
                        .source(GsonUtil.toJSONBytes(metric), XContentType.JSON));
            },
            parameterTool);

    env.execute("flink learning connectors es7");
}

Example #8

Source File: StreamExecutionEnvironment.java From flink with Apache License 2.0

1 votes

/**
 * Ads a data source with a custom type information thus opening a
 * {@link DataStream}. Only in very special cases does the user need to
 * support type information. Otherwise use
 * {@link #addSource(org.apache.flink.streaming.api.functions.source.SourceFunction)}
 *
 * @param function
 * 		the user defined function
 * @param sourceName
 * 		Name of the data source
 * @param <OUT>
 * 		type of the returned stream
 * @param typeInfo
 * 		the user defined type information for the stream
 * @return the data stream constructed
 */
public <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> function, String sourceName, TypeInformation<OUT> typeInfo) {

	TypeInformation<OUT> resolvedTypeInfo = getTypeInfo(function, sourceName, SourceFunction.class, typeInfo);

	boolean isParallel = function instanceof ParallelSourceFunction;

	clean(function);

	final StreamSource<OUT, ?> sourceOperator = new StreamSource<>(function);
	return new DataStreamSource<>(this, resolvedTypeInfo, sourceOperator, isParallel, sourceName);
}