Java Code Examples for org.apache.flink.api.java.ExecutionEnvironment#fromCollection()

The following examples show how to use org.apache.flink.api.java.ExecutionEnvironment#fromCollection() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: DataSinkTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testPojoTwoOrder() {

	final ExecutionEnvironment env = ExecutionEnvironment
			.getExecutionEnvironment();
	DataSet<CustomType> pojoDs = env
			.fromCollection(pojoData);

	// should work
	try {
		pojoDs.writeAsText("/tmp/willNotHappen")
			.sortLocalOutput("myLong", Order.ASCENDING)
			.sortLocalOutput("myString", Order.DESCENDING);
	} catch (Exception e) {
		Assert.fail();
	}
}

Example 2

Source File: ValueCollectionDataSets.java From flink with Apache License 2.0

6 votes

public static DataSet<Tuple3<Tuple2<IntValue, IntValue>, StringValue, IntValue>> getGroupSortedNestedTupleDataSet2(ExecutionEnvironment env) {
	List<Tuple3<Tuple2<IntValue, IntValue>, StringValue, IntValue>> data = new ArrayList<>();

	data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(1), new IntValue(3)), new StringValue("a"), new IntValue(2)));
	data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(1), new IntValue(2)), new StringValue("a"), new IntValue(1)));
	data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(2), new IntValue(1)), new StringValue("a"), new IntValue(3)));
	data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(2), new IntValue(2)), new StringValue("b"), new IntValue(4)));
	data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(3), new IntValue(3)), new StringValue("c"), new IntValue(5)));
	data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(3), new IntValue(6)), new StringValue("c"), new IntValue(6)));
	data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(4), new IntValue(9)), new StringValue("c"), new IntValue(7)));

	TupleTypeInfo<Tuple3<Tuple2<IntValue, IntValue>, StringValue, IntValue>> type = new
		TupleTypeInfo<>(
			new TupleTypeInfo<Tuple2<IntValue, IntValue>>(ValueTypeInfo.INT_VALUE_TYPE_INFO, ValueTypeInfo.INT_VALUE_TYPE_INFO),
			ValueTypeInfo.STRING_VALUE_TYPE_INFO,
			ValueTypeInfo.INT_VALUE_TYPE_INFO
	);

	return env.fromCollection(data, type);
}

Example 3

Source File: SortPartitionTest.java From flink with Apache License 2.0

6 votes

@Test
public void testSortPartitionWithKeySelector4() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs = env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);

	// should work
	try {
		tupleDs.sortPartition(new KeySelector<Tuple4<Integer, Long, CustomType, Long[]>, Tuple2<Integer, Long>>() {
			@Override
			public Tuple2<Integer, Long> getKey(Tuple4<Integer, Long, CustomType, Long[]> value) throws Exception {
				return new Tuple2<>(value.f0, value.f1);
			}
		}, Order.ASCENDING);
	} catch (Exception e) {
		Assert.fail();
	}
}

Example 4

Source File: TestGraphUtils.java From flink with Apache License 2.0

5 votes

public static DataSet<Tuple3<Long, Long, Long>> getLongLongLongTuple3Data(
		ExecutionEnvironment env) {
	List<Tuple3<Long, Long, Long>> tuples = new ArrayList<>();
	tuples.add(new Tuple3<>(1L, 2L, 12L));
	tuples.add(new Tuple3<>(1L, 3L, 13L));
	tuples.add(new Tuple3<>(2L, 3L, 23L));
	tuples.add(new Tuple3<>(3L, 4L, 34L));
	tuples.add(new Tuple3<>(3L, 6L, 36L));
	tuples.add(new Tuple3<>(4L, 6L, 46L));
	tuples.add(new Tuple3<>(6L, 1L, 61L));

	return env.fromCollection(tuples);
}

Example 5

Source File: CrossOperatorTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test(expected = IndexOutOfBoundsException.class)
public void testCrossProjection9() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);

	// should not work, index out of range
	ds1.cross(ds2)
		.projectSecond(5);
}

Example 6

Source File: PartitionOperatorTest.java From flink with Apache License 2.0

5 votes

@Test
public void testRangePartitionOperatorPreservesFields() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

		DataSet<Tuple2<Long, Long>> data = env.fromCollection(Collections.singleton(new Tuple2<>(0L, 0L)));

		data.partitionByRange(1)
			.groupBy(1)
			.reduceGroup(new IdentityGroupReducerCombinable<Tuple2<Long,Long>>())
			.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());

		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);

		SinkPlanNode sink = op.getDataSinks().iterator().next();
		SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
		SingleInputPlanNode partitionNode = (SingleInputPlanNode)reducer.getInput().getSource();
		SingleInputPlanNode partitionIDRemover = (SingleInputPlanNode) partitionNode.getInput().getSource();

		assertEquals(ShipStrategyType.FORWARD, reducer.getInput().getShipStrategy());
		assertEquals(ShipStrategyType.FORWARD, partitionNode.getInput().getShipStrategy());
		assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitionIDRemover.getInput().getShipStrategy());

		SourcePlanNode sourcePlanNode = op.getDataSources().iterator().next();
		List<Channel> sourceOutgoingChannels = sourcePlanNode.getOutgoingChannels();
		assertEquals(2, sourceOutgoingChannels.size());
		assertEquals(ShipStrategyType.FORWARD, sourceOutgoingChannels.get(0).getShipStrategy());
		assertEquals(ShipStrategyType.FORWARD, sourceOutgoingChannels.get(1).getShipStrategy());
		assertEquals(DataExchangeMode.PIPELINED, sourceOutgoingChannels.get(0).getDataExchangeMode());
		assertEquals(DataExchangeMode.BATCH, sourceOutgoingChannels.get(1).getDataExchangeMode());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example 7

Source File: IncrementalSSSPData.java From flink with Apache License 2.0

5 votes

public static DataSet<Vertex<Long, Double>> getDefaultVertexDataSet(ExecutionEnvironment env) {

		List<Vertex<Long, Double>> vertices = new ArrayList<>();
		vertices.add(new Vertex<>(1L, 6.0));
		vertices.add(new Vertex<>(2L, 2.0));
		vertices.add(new Vertex<>(3L, 3.0));
		vertices.add(new Vertex<>(4L, 1.0));
		vertices.add(new Vertex<>(5L, 0.0));

		return env.fromCollection(vertices);
	}

Example 8

Source File: DistinctOperatorTest.java From flink with Apache License 2.0

5 votes

@Test(expected = InvalidProgramException.class)
public void testDistinctByKeyFields2() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Long> longDs = env.fromCollection(emptyLongData, BasicTypeInfo.LONG_TYPE_INFO);
	// should not work: distinct on basic type
	longDs.distinct(0);
}

Example 9

Source File: CoGroupOperatorTest.java From flink with Apache License 2.0

5 votes

@Test(expected = IndexOutOfBoundsException.class)
public void testCoGroupKeyFields5() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);

	// should not work, negative key field position
	ds1.coGroup(ds2).where(-1).equalTo(-1);
}

Example 10

Source File: JoinOperatorTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test(expected = InvalidProgramException.class)
public void testJoinKeyInvalidAtomic4() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<Integer> ds2 = env.fromElements(0, 0, 0);

	ds1.join(ds2).where(0).equalTo("invalidKey");
}

Example 11

Source File: DataSinkTest.java From flink with Apache License 2.0

5 votes

@Test
public void testTupleSingleOrderIdx() {

	final ExecutionEnvironment env = ExecutionEnvironment
			.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env
			.fromCollection(emptyTupleData, tupleTypeInfo);

	// should work
	try {
		tupleDs.writeAsText("/tmp/willNotHappen").sortLocalOutput(0, Order.ANY);
	} catch (Exception e) {
		Assert.fail();
	}
}

Example 12

Source File: TestGraphUtils.java From flink with Apache License 2.0

5 votes

public static DataSet<Edge<String, Long>> getStringLongEdgeData(
		ExecutionEnvironment env) {
	List<Edge<String, Long>> edges = new ArrayList<>();
	edges.add(new Edge<>("1", "2", 12L));
	edges.add(new Edge<>("1", "3", 13L));
	edges.add(new Edge<>("2", "3", 23L));
	edges.add(new Edge<>("3", "4", 34L));
	edges.add(new Edge<>("3", "5", 35L));
	edges.add(new Edge<>("4", "5", 45L));
	edges.add(new Edge<>("5", "1", 51L));
	return env.fromCollection(edges);
}

Example 13

Source File: TestGraphUtils.java From flink with Apache License 2.0

5 votes

public static DataSet<Edge<Long, Long>> getDisconnectedLongLongEdgeData(ExecutionEnvironment env) {
	List<Edge<Long, Long>> edges = new ArrayList<>();
	edges.add(new Edge<>(1L, 2L, 12L));
	edges.add(new Edge<>(1L, 3L, 13L));
	edges.add(new Edge<>(2L, 3L, 23L));
	edges.add(new Edge<>(4L, 5L, 45L));

	return env.fromCollection(edges);
}

Example 14

Source File: GroupingTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test(expected = InvalidProgramException.class)
public void testGroupSortKeyFields5() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs =
			env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);

	// should not work
	tupleDs.groupBy(0)
			.sortGroup(3, Order.ASCENDING);
}

Example 15

Source File: CrossOperatorTest.java From Flink-CEPplus with Apache License 2.0

5 votes

public void testCrossProjection12() {

		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
		DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);

		// should work
		ds1.cross(ds2)
			.projectSecond(2)
			.projectFirst(1);
	}

Example 16

Source File: ValueCollectionDataSets.java From flink with Apache License 2.0

5 votes

private static DataSet<Tuple7<LongValue, IntValue, IntValue, LongValue, StringValue, IntValue, StringValue>> getSmallTuplebasedDataSetMatchingPojo(ExecutionEnvironment env) {
	List<Tuple7<LongValue, IntValue, IntValue, LongValue, StringValue, IntValue, StringValue>> data = new ArrayList<>();

	data.add(new Tuple7<>(new LongValue(10000L), new IntValue(10), new IntValue(100), new LongValue(1000L), new StringValue("One"), new IntValue(1), new StringValue("First")));
	data.add(new Tuple7<>(new LongValue(20000L), new IntValue(20), new IntValue(200), new LongValue(2000L), new StringValue("Two"), new IntValue(2), new StringValue("Second")));
	data.add(new Tuple7<>(new LongValue(30000L), new IntValue(30), new IntValue(300), new LongValue(3000L), new StringValue("Three"), new IntValue(3), new StringValue("Third")));

	return env.fromCollection(data);
}

Example 17

Source File: GroupingTest.java From flink with Apache License 2.0

5 votes

@Test(expected = InvalidProgramException.class)
public void testGroupSortKeyFields4() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs =
			env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);

	// should not work
	tupleDs.groupBy(0)
			.sortGroup(2, Order.ASCENDING);
}

Example 18

Source File: ValueCollectionDataSets.java From flink with Apache License 2.0

5 votes

public static DataSet<PojoContainingTupleAndWritable> getGroupSortedPojoContainingTupleAndWritable(ExecutionEnvironment env) {
	List<PojoContainingTupleAndWritable> data = new ArrayList<>();
	data.add(new PojoContainingTupleAndWritable(1, 10L, 100L)); // 1x
	data.add(new PojoContainingTupleAndWritable(2, 20L, 200L)); // 5x
	data.add(new PojoContainingTupleAndWritable(2, 20L, 201L));
	data.add(new PojoContainingTupleAndWritable(2, 30L, 200L));
	data.add(new PojoContainingTupleAndWritable(2, 30L, 600L));
	data.add(new PojoContainingTupleAndWritable(2, 30L, 400L));
	return env.fromCollection(data);
}

Example 19

Source File: CrossOperatorTest.java From flink with Apache License 2.0

5 votes

public void testCrossProjection12() {

		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
		DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);

		// should work
		ds1.cross(ds2)
			.projectSecond(2)
			.projectFirst(1);
	}

Example 20

Source File: TestGraphUtils.java From flink with Apache License 2.0

4 votes

public static DataSet<Edge<Long, Long>> getLongLongEdgeData(
		ExecutionEnvironment env) {

	return env.fromCollection(getLongLongEdges());
}