Java Code Examples for org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#createLocalEnvironment()

/**
 * Tests that a changed operator name does not affect the hash.
 */
@Test
public void testChangedOperatorName() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.addSource(new NoOpSourceFunction(), "A").map(new NoOpMapFunction());
	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	JobVertexID expected = jobGraph.getVerticesAsArray()[0].getID();

	env = StreamExecutionEnvironment.createLocalEnvironment();
	env.addSource(new NoOpSourceFunction(), "B").map(new NoOpMapFunction());
	jobGraph = env.getStreamGraph().getJobGraph();

	JobVertexID actual = jobGraph.getVerticesAsArray()[0].getID();

	assertEquals(expected, actual);
}
 

/**
 * Tests that a changed operator name does not affect the hash.
 */
@Test
public void testChangedOperatorName() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.addSource(new NoOpSourceFunction(), "A").map(new NoOpMapFunction());
	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	JobVertexID expected = jobGraph.getVerticesAsArray()[0].getID();

	env = StreamExecutionEnvironment.createLocalEnvironment();
	env.addSource(new NoOpSourceFunction(), "B").map(new NoOpMapFunction());
	jobGraph = env.getStreamGraph().getJobGraph();

	JobVertexID actual = jobGraph.getVerticesAsArray()[0].getID();

	assertEquals(expected, actual);
}
 

@Before
public void init()
        throws JsonProcessingException
{
    String json = MAPPER.writeValueAsString(ImmutableMap.of("user_id", "uid_001",
            "ip", "127.0.0.1",
            "store", 12.0,
            "key1", ImmutableMap.of("key2", 123)
    ));

    StreamExecutionEnvironment execEnv = StreamExecutionEnvironment.createLocalEnvironment();
    execEnv.setParallelism(2);
    tableEnv = StreamTableEnvironment.create(execEnv);
    tableEnv.registerFunction("get_json_object", new UDFJson());
    table = tableEnv.sqlQuery("select '" + json + "' as message");
}
 

/**
 * Tests that there are no collisions with two identical sources.
 *
 * <pre>
 * [ (src0) ] --\
 *               +--> [ (sink) ]
 * [ (src1) ] --/
 * </pre>
 */
@Test
public void testNodeHashIdenticalSources() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);
	env.disableOperatorChaining();

	DataStream<String> src0 = env.addSource(new NoOpSourceFunction());
	DataStream<String> src1 = env.addSource(new NoOpSourceFunction());

	src0.union(src1).addSink(new DiscardingSink<>());

	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();
	assertTrue(vertices.get(0).isInputVertex());
	assertTrue(vertices.get(1).isInputVertex());

	assertNotNull(vertices.get(0).getID());
	assertNotNull(vertices.get(1).getID());

	assertNotEquals(vertices.get(0).getID(), vertices.get(1).getID());
}
 

@Test
public void testYieldingOperatorChainableToTaskNotChainedToLegacySource() {
	StreamExecutionEnvironment chainEnv = StreamExecutionEnvironment.createLocalEnvironment(1);

	chainEnv.fromElements(1).disableChaining()
		.map((x) -> x)
		.transform("test", BasicTypeInfo.INT_TYPE_INFO, new YieldingTestOperatorFactory<>());

	final StreamGraph streamGraph = chainEnv.getStreamGraph();

	final List<StreamNode> streamNodes = streamGraph.getStreamNodes().stream()
		.sorted(Comparator.comparingInt(StreamNode::getId))
		.collect(Collectors.toList());
	assertFalse(areOperatorsChainable(streamNodes.get(0), streamNodes.get(1), streamGraph));
	assertTrue(areOperatorsChainable(streamNodes.get(1), streamNodes.get(2), streamGraph));
}
 

@Test
public void testEventTimeOrderedWriter() throws Exception {
    StreamExecutionEnvironment execEnv = StreamExecutionEnvironment.createLocalEnvironment();

    Stream stream = Stream.of(SETUP_UTILS.getScope(), "testEventTimeOrderedWriter");
    SETUP_UTILS.createTestStream(stream.getStreamName(), 1);

    DataStreamSource<Integer> dataStream = execEnv
            .addSource(new IntegerGeneratingSource(false, EVENT_COUNT_PER_SOURCE));

    FlinkPravegaWriter<Integer> pravegaSink = FlinkPravegaWriter.<Integer>builder()
            .withPravegaConfig(SETUP_UTILS.getPravegaConfig())
            .forStream(stream)
            .withSerializationSchema(new IntSerializer())
            .withEventRouter(event -> "fixedkey")
            .build();

    FlinkPravegaUtils.writeToPravegaInEventTimeOrder(dataStream, pravegaSink, 1);
    Assert.assertNotNull(execEnv.getExecutionPlan());
}
 

public static StreamTableEnvironment getTableEnv()
{
    StreamExecutionEnvironment execEnv = StreamExecutionEnvironment.createLocalEnvironment();
    execEnv.setParallelism(2);
    StreamTableEnvironment tableEnv = TableEnvironment.getTableEnvironment(execEnv);
    return tableEnv;
}
 

/**
 * If expected values ever change double check that the change is not braking the contract of
 * {@link StreamingRuntimeContext#getOperatorUniqueID()} being stable between job submissions.
 */
@Test
public void testGetOperatorUniqueID() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();

	env.fromElements(1, 2, 3)
		.map(new VerifyOperatorIDMapFunction("6c4f323f22da8fb6e34f80c61be7a689")).uid("42")
		.map(new VerifyOperatorIDMapFunction("3e129e83691e7737fbf876b47452acbc")).uid("44");

	env.execute();
}
 

@Test
public void testUserProvidedHashingOnChainSupported() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();

	env.addSource(new NoOpSourceFunction(), "src").setUidHash("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
			.map(new NoOpMapFunction()).setUidHash("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
			.filter(new NoOpFilterFunction()).setUidHash("cccccccccccccccccccccccccccccccc")
			.keyBy(new NoOpKeySelector())
			.reduce(new NoOpReduceFunction()).name("reduce").setUidHash("dddddddddddddddddddddddddddddddd");

	env.getStreamGraph().getJobGraph();
}
 

@Test
public void testUserProvidedHashingOnChainSupported() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();

	env.addSource(new NoOpSourceFunction(), "src").setUidHash("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
			.map(new NoOpMapFunction()).setUidHash("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
			.filter(new NoOpFilterFunction()).setUidHash("cccccccccccccccccccccccccccccccc")
			.keyBy(new NoOpKeySelector())
			.reduce(new NoOpReduceFunction()).name("reduce").setUidHash("dddddddddddddddddddddddddddddddd");

	env.getStreamGraph().getJobGraph();
}
 

/**
 * Tests that a manual hash for an intermediate chain node is accepted.
 */
@Test
public void testManualHashAssignmentForIntermediateNodeInChain() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);

	env.addSource(new NoOpSourceFunction())
			// Intermediate chained node
			.map(new NoOpMapFunction()).uid("map")
			.addSink(new NoOpSinkFunction());

	env.getStreamGraph().getJobGraph();
}
 

@Test
public void testDisablingAutoUidsAcceptsManuallySetId() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.getConfig().disableAutoGeneratedUIDs();

	env
		.addSource(new NoOpSourceFunction()).uid("uid1")
		.addSink(new DiscardingSink<>()).uid("uid2");

	env.getStreamGraph();
}
 

/**
 * Tests that (un)chaining affects the node hash (for intermediate nodes).
 *
 * <pre>
 * A (chained): [ (src0) -> (map) -> (filter) -> (sink) ]
 * B (unchained): [ (src0) ] -> [ (map) -> (filter) -> (sink) ]
 * </pre>
 *
 * <p>The hashes for the single vertex in A and the source vertex in B need to be different.
 */
@Test
public void testNodeHashAfterIntermediateUnchaining() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);

	env.addSource(new NoOpSourceFunction())
			.map(new NoOpMapFunction()).name("map")
			.startNewChain()
			.filter(new NoOpFilterFunction())
			.addSink(new NoOpSinkFunction());

	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	JobVertex chainedMap = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
	assertTrue(chainedMap.getName().startsWith("map"));
	JobVertexID chainedMapId = chainedMap.getID();

	env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);

	env.addSource(new NoOpSourceFunction())
			.map(new NoOpMapFunction()).name("map")
			.startNewChain()
			.filter(new NoOpFilterFunction())
			.startNewChain()
			.addSink(new NoOpSinkFunction());

	jobGraph = env.getStreamGraph().getJobGraph();

	JobVertex unchainedMap = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
	assertEquals("map", unchainedMap.getName());
	JobVertexID unchainedMapId = unchainedMap.getID();

	assertNotEquals(chainedMapId, unchainedMapId);
}
 

/**
 * Tests that a collision on the manual hash throws an Exception.
 */
@Test(expected = IllegalArgumentException.class)
public void testManualHashAssignmentCollisionThrowsException() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);
	env.disableOperatorChaining();

	env.addSource(new NoOpSourceFunction()).uid("source")
			.map(new NoOpMapFunction()).uid("source") // Collision
			.addSink(new DiscardingSink<>());

	// This call is necessary to generate the job graph
	env.getStreamGraph().getJobGraph();
}
 

public CentralizedWeightedMatching() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();

	// Source: http://grouplens.org/datasets/movielens/
	@SuppressWarnings("serial")
	DataStream<Edge<Long, Long>> edges = env
			.readTextFile("movielens_10k_sorted.txt")
			.map(new MapFunction<String, Edge<Long, Long>>() {
				@Override
				public Edge<Long, Long> map(String s) throws Exception {
					String[] args = s.split("\t");
					long src = Long.parseLong(args[0]);
					long trg = Long.parseLong(args[1]) + 1000000;
					long val = Long.parseLong(args[2]) * 10;
					return new Edge<>(src, trg, val);
				}
			});

	GraphStream<Long, NullValue, Long> graph = new SimpleEdgeStream<>(edges, env);

	graph.getEdges()
			.flatMap(new WeightedMatchingFlatMapper()).setParallelism(1)
			.print().setParallelism(1);

	JobExecutionResult res = env.execute("Distributed Merge Tree Sandbox");
	long runtime = res.getNetRuntime();
	System.out.println("Runtime: " + runtime);
}
 

@Test(expected = IllegalStateException.class)
public void testDisablingAutoUidsFailsStreamGraphCreation() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.getConfig().disableAutoGeneratedUIDs();

	env.addSource(new NoOpSourceFunction()).addSink(new DiscardingSink<>());
	env.getStreamGraph();
}
 

/**
 * Tests that a manual hash at the beginning of a chain is accepted.
 */
@Test
public void testManualHashAssignmentForStartNodeInInChain() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);

	env.addSource(new NoOpSourceFunction()).uid("source")
			.map(new NoOpMapFunction())
			.addSink(new DiscardingSink<>());

	env.getStreamGraph().getJobGraph();
}
 

@Bean("flinkEnvironment")
@Primary
StreamExecutionEnvironment getFlinkEnvironment(FlinkProperties flinkProperties) {
    return StreamExecutionEnvironment.createLocalEnvironment();
}
 

/**
 * Creates the same flow twice and checks that all IDs are the same.
 *
 * <pre>
 * [ (src) -> (map) -> (filter) -> (reduce) -> (map) -> (sink) ]
 *                                                       //
 * [ (src) -> (filter) ] -------------------------------//
 *                                                      /
 * [ (src) -> (filter) ] ------------------------------/
 * </pre>
 */
@Test
public void testNodeHashIsDeterministic() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);

	DataStream<String> src0 = env
			.addSource(new NoOpSourceFunction(), "src0")
			.map(new NoOpMapFunction())
			.filter(new NoOpFilterFunction())
			.keyBy(new NoOpKeySelector())
			.reduce(new NoOpReduceFunction()).name("reduce");

	DataStream<String> src1 = env
			.addSource(new NoOpSourceFunction(), "src1")
			.filter(new NoOpFilterFunction());

	DataStream<String> src2 = env
			.addSource(new NoOpSourceFunction(), "src2")
			.filter(new NoOpFilterFunction());

	src0.map(new NoOpMapFunction())
			.union(src1, src2)
			.addSink(new DiscardingSink<>()).name("sink");

	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	final Map<JobVertexID, String> ids = rememberIds(jobGraph);

	// Do it again and verify
	env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);

	src0 = env
			.addSource(new NoOpSourceFunction(), "src0")
			.map(new NoOpMapFunction())
			.filter(new NoOpFilterFunction())
			.keyBy(new NoOpKeySelector())
			.reduce(new NoOpReduceFunction()).name("reduce");

	src1 = env
			.addSource(new NoOpSourceFunction(), "src1")
			.filter(new NoOpFilterFunction());

	src2 = env
			.addSource(new NoOpSourceFunction(), "src2")
			.filter(new NoOpFilterFunction());

	src0.map(new NoOpMapFunction())
			.union(src1, src2)
			.addSink(new DiscardingSink<>()).name("sink");

	jobGraph = env.getStreamGraph().getJobGraph();

	verifyIdsEqual(jobGraph, ids);
}
 

private StreamExecutionEnvironment getStreamExecutionEnvironment() {
    return StreamExecutionEnvironment.createLocalEnvironment(1, createConfiguration());
}