Java Code Examples for org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#generateSequence()

The following examples show how to use org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#generateSequence() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: CollectITCase.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testCollect() throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);

	final long n = 10;
	DataStream<Long> stream = env.generateSequence(1, n);

	long i = 1;
	for (Iterator<Long> it = DataStreamUtils.collect(stream); it.hasNext(); ) {
		long x = it.next();
		assertEquals("received wrong element", i, x);
		i++;
	}

	assertEquals("received wrong number of elements", n + 1, i);
}

Example 2

Source File: CollectITCase.java From flink with Apache License 2.0

6 votes

@Test
public void testCollect() throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);

	final long n = 10;
	DataStream<Long> stream = env.generateSequence(1, n);

	long i = 1;
	for (Iterator<Long> it = DataStreamUtils.collect(stream); it.hasNext(); ) {
		long x = it.next();
		assertEquals("received wrong element", i, x);
		i++;
	}

	assertEquals("received wrong number of elements", n + 1, i);
}

Example 3

Source File: GenerateSequence.java From blog_demos with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    //并行度为1
    env.setParallelism(1);

    //通过generateSequence得到Long类型的DataSource
    DataStream<Long> dataStream = env.generateSequence(1, 10);

    //做一次过滤，只保留偶数，然后打印
    dataStream.filter(new FilterFunction<Long>() {
        @Override
        public boolean filter(Long aLong) throws Exception {
            return 0L==aLong.longValue()%2L;
        }
    }).print();


    env.execute("API DataSource demo : collection");
}

Example 4

Source File: CollectITCase.java From flink with Apache License 2.0

6 votes

@Test
public void testCollect() throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);

	final long n = 10;
	DataStream<Long> stream = env.generateSequence(1, n);

	long i = 1;
	for (Iterator<Long> it = DataStreamUtils.collect(stream); it.hasNext(); ) {
		long x = it.next();
		assertEquals("received wrong element", i, x);
		i++;
	}

	assertEquals("received wrong number of elements", n + 1, i);
}

Example 5

Source File: StreamExecutionEnvironmentTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testSources() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	SourceFunction<Integer> srcFun = new SourceFunction<Integer>() {
		private static final long serialVersionUID = 1L;

		@Override
		public void run(SourceContext<Integer> ctx) throws Exception {
		}

		@Override
		public void cancel() {
		}
	};
	DataStreamSource<Integer> src1 = env.addSource(srcFun);
	src1.addSink(new DiscardingSink<Integer>());
	assertEquals(srcFun, getFunctionFromDataSource(src1));

	List<Long> list = Arrays.asList(0L, 1L, 2L);

	DataStreamSource<Long> src2 = env.generateSequence(0, 2);
	assertTrue(getFunctionFromDataSource(src2) instanceof StatefulSequenceSource);

	DataStreamSource<Long> src3 = env.fromElements(0L, 1L, 2L);
	assertTrue(getFunctionFromDataSource(src3) instanceof FromElementsFunction);

	DataStreamSource<Long> src4 = env.fromCollection(list);
	assertTrue(getFunctionFromDataSource(src4) instanceof FromElementsFunction);
}

Example 6

Source File: SlotAllocationTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testUnion() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	FilterFunction<Long> dummyFilter = new FilterFunction<Long>() {
		@Override
		public boolean filter(Long value) {
			return false;
		}
	};

	DataStream<Long> src1 = env.generateSequence(1, 10);
	DataStream<Long> src2 = env.generateSequence(1, 10).slotSharingGroup("src-1");

	// this should not inherit group "src-1"
	src1.union(src2).filter(dummyFilter);

	DataStream<Long> src3 = env.generateSequence(1, 10).slotSharingGroup("group-1");
	DataStream<Long> src4 = env.generateSequence(1, 10).slotSharingGroup("group-1");

	// this should inherit "group-1" now
	src3.union(src4).filter(dummyFilter);

	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();

	// first pipeline
	assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup());
	assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(1).getSlotSharingGroup());
	assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup());

	// second pipeline
	assertEquals(vertices.get(2).getSlotSharingGroup(), vertices.get(3).getSlotSharingGroup());
	assertEquals(vertices.get(2).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup());
	assertEquals(vertices.get(3).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup());
}

Example 7

Source File: StreamExecutionEnvironmentTest.java From flink with Apache License 2.0

5 votes

@Test
public void testSources() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	SourceFunction<Integer> srcFun = new SourceFunction<Integer>() {
		private static final long serialVersionUID = 1L;

		@Override
		public void run(SourceContext<Integer> ctx) throws Exception {
		}

		@Override
		public void cancel() {
		}
	};
	DataStreamSource<Integer> src1 = env.addSource(srcFun);
	src1.addSink(new DiscardingSink<Integer>());
	assertEquals(srcFun, getFunctionFromDataSource(src1));

	List<Long> list = Arrays.asList(0L, 1L, 2L);

	DataStreamSource<Long> src2 = env.generateSequence(0, 2);
	assertTrue(getFunctionFromDataSource(src2) instanceof StatefulSequenceSource);

	DataStreamSource<Long> src3 = env.fromElements(0L, 1L, 2L);
	assertTrue(getFunctionFromDataSource(src3) instanceof FromElementsFunction);

	DataStreamSource<Long> src4 = env.fromCollection(list);
	assertTrue(getFunctionFromDataSource(src4) instanceof FromElementsFunction);
}

Example 8

Source File: SlotAllocationTest.java From flink with Apache License 2.0

5 votes

@Test
public void testUnion() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	FilterFunction<Long> dummyFilter = new FilterFunction<Long>() {
		@Override
		public boolean filter(Long value) {
			return false;
		}
	};

	DataStream<Long> src1 = env.generateSequence(1, 10);
	DataStream<Long> src2 = env.generateSequence(1, 10).slotSharingGroup("src-1");

	// this should not inherit group "src-1"
	src1.union(src2).filter(dummyFilter);

	DataStream<Long> src3 = env.generateSequence(1, 10).slotSharingGroup("group-1");
	DataStream<Long> src4 = env.generateSequence(1, 10).slotSharingGroup("group-1");

	// this should inherit "group-1" now
	src3.union(src4).filter(dummyFilter);

	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();

	// first pipeline
	assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup());
	assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(1).getSlotSharingGroup());
	assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup());

	// second pipeline
	assertEquals(vertices.get(2).getSlotSharingGroup(), vertices.get(3).getSlotSharingGroup());
	assertEquals(vertices.get(2).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup());
	assertEquals(vertices.get(3).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup());
}

Example 9

Source File: StreamExecutionEnvironmentTest.java From flink with Apache License 2.0

5 votes

@Test
public void testSources() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	SourceFunction<Integer> srcFun = new SourceFunction<Integer>() {
		private static final long serialVersionUID = 1L;

		@Override
		public void run(SourceContext<Integer> ctx) throws Exception {
		}

		@Override
		public void cancel() {
		}
	};
	DataStreamSource<Integer> src1 = env.addSource(srcFun);
	src1.addSink(new DiscardingSink<Integer>());
	assertEquals(srcFun, getFunctionFromDataSource(src1));

	List<Long> list = Arrays.asList(0L, 1L, 2L);

	DataStreamSource<Long> src2 = env.generateSequence(0, 2);
	assertTrue(getFunctionFromDataSource(src2) instanceof StatefulSequenceSource);

	DataStreamSource<Long> src3 = env.fromElements(0L, 1L, 2L);
	assertTrue(getFunctionFromDataSource(src3) instanceof FromElementsFunction);

	DataStreamSource<Long> src4 = env.fromCollection(list);
	assertTrue(getFunctionFromDataSource(src4) instanceof FromElementsFunction);
}

Example 10

Source File: SlotAllocationTest.java From flink with Apache License 2.0

5 votes

@Test
public void testUnion() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	FilterFunction<Long> dummyFilter = new FilterFunction<Long>() {
		@Override
		public boolean filter(Long value) {
			return false;
		}
	};

	DataStream<Long> src1 = env.generateSequence(1, 10);
	DataStream<Long> src2 = env.generateSequence(1, 10).slotSharingGroup("src-1");

	// this should not inherit group "src-1"
	src1.union(src2).filter(dummyFilter);

	DataStream<Long> src3 = env.generateSequence(1, 10).slotSharingGroup("group-1");
	DataStream<Long> src4 = env.generateSequence(1, 10).slotSharingGroup("group-1");

	// this should inherit "group-1" now
	src3.union(src4).filter(dummyFilter);

	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();

	// first pipeline
	assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup());
	assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(1).getSlotSharingGroup());
	assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup());

	// second pipeline
	assertEquals(vertices.get(2).getSlotSharingGroup(), vertices.get(3).getSlotSharingGroup());
	assertEquals(vertices.get(2).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup());
	assertEquals(vertices.get(3).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup());
}

Example 11

Source File: DataStreamTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testChannelSelectors() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStreamSource<Long> src = env.generateSequence(0, 0);

	DataStream<Long> broadcast = src.broadcast();
	DataStreamSink<Long> broadcastSink = broadcast.print();
	StreamPartitioner<?> broadcastPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					broadcastSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);

	DataStream<Long> shuffle = src.shuffle();
	DataStreamSink<Long> shuffleSink = shuffle.print();
	StreamPartitioner<?> shufflePartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					shuffleSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(shufflePartitioner instanceof ShufflePartitioner);

	DataStream<Long> forward = src.forward();
	DataStreamSink<Long> forwardSink = forward.print();
	StreamPartitioner<?> forwardPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					forwardSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(forwardPartitioner instanceof ForwardPartitioner);

	DataStream<Long> rebalance = src.rebalance();
	DataStreamSink<Long> rebalanceSink = rebalance.print();
	StreamPartitioner<?> rebalancePartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					rebalanceSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(rebalancePartitioner instanceof RebalancePartitioner);

	DataStream<Long> global = src.global();
	DataStreamSink<Long> globalSink = global.print();
	StreamPartitioner<?> globalPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					globalSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(globalPartitioner instanceof GlobalPartitioner);
}

Example 12

Source File: DataStreamTest.java From flink with Apache License 2.0

4 votes

@Test
public void testChannelSelectors() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStreamSource<Long> src = env.generateSequence(0, 0);

	DataStream<Long> broadcast = src.broadcast();
	DataStreamSink<Long> broadcastSink = broadcast.print();
	StreamPartitioner<?> broadcastPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					broadcastSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);

	DataStream<Long> shuffle = src.shuffle();
	DataStreamSink<Long> shuffleSink = shuffle.print();
	StreamPartitioner<?> shufflePartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					shuffleSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(shufflePartitioner instanceof ShufflePartitioner);

	DataStream<Long> forward = src.forward();
	DataStreamSink<Long> forwardSink = forward.print();
	StreamPartitioner<?> forwardPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					forwardSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(forwardPartitioner instanceof ForwardPartitioner);

	DataStream<Long> rebalance = src.rebalance();
	DataStreamSink<Long> rebalanceSink = rebalance.print();
	StreamPartitioner<?> rebalancePartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					rebalanceSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(rebalancePartitioner instanceof RebalancePartitioner);

	DataStream<Long> global = src.global();
	DataStreamSink<Long> globalSink = global.print();
	StreamPartitioner<?> globalPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					globalSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(globalPartitioner instanceof GlobalPartitioner);
}

Example 13

Source File: DataStreamTest.java From flink with Apache License 2.0

4 votes

@Test
public void testChannelSelectors() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStreamSource<Long> src = env.generateSequence(0, 0);

	DataStream<Long> broadcast = src.broadcast();
	DataStreamSink<Long> broadcastSink = broadcast.print();
	StreamPartitioner<?> broadcastPartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					broadcastSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);

	DataStream<Long> shuffle = src.shuffle();
	DataStreamSink<Long> shuffleSink = shuffle.print();
	StreamPartitioner<?> shufflePartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					shuffleSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(shufflePartitioner instanceof ShufflePartitioner);

	DataStream<Long> forward = src.forward();
	DataStreamSink<Long> forwardSink = forward.print();
	StreamPartitioner<?> forwardPartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					forwardSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(forwardPartitioner instanceof ForwardPartitioner);

	DataStream<Long> rebalance = src.rebalance();
	DataStreamSink<Long> rebalanceSink = rebalance.print();
	StreamPartitioner<?> rebalancePartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					rebalanceSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(rebalancePartitioner instanceof RebalancePartitioner);

	DataStream<Long> global = src.global();
	DataStreamSink<Long> globalSink = global.print();
	StreamPartitioner<?> globalPartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					globalSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(globalPartitioner instanceof GlobalPartitioner);
}