Java Code Examples for org.apache.flink.api.java.ExecutionEnvironment#getExecutionEnvironment()

The following examples show how to use org.apache.flink.api.java.ExecutionEnvironment#getExecutionEnvironment() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DistinctITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testCorrectnessOfDistinctOnAtomic() throws Exception {
	/*
	 * check correctness of distinct on Integers
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Integer> ds = CollectionDataSets.getIntegerDataSet(env);
	DataSet<Integer> reduceDs = ds.distinct();

	List<Integer> result = reduceDs.collect();

	String expected = "1\n2\n3\n4\n5";

	compareResultAsText(result, expected);
}
 
Example 2
Source File: ReduceWithCombinerITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testReduceOnKeyedDatasetWithSelector() throws Exception {

	// set up the execution environment
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(4);

	// creates the input data and distributes them evenly among the available downstream tasks
	DataSet<Tuple3<String, Integer, Boolean>> input = createKeyedInput(env);

	List<Tuple3<String, Integer, Boolean>> actual = input
		.groupBy(new KeySelectorX())
		.reduceGroup(new KeyedCombReducer())
		.collect();
	String expected = "k1,6,true\nk2,4,true\n";

	compareResultAsTuples(actual, expected);
}
 
Example 3
Source File: JavaSqlITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testFilterFromDataSet() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	tableEnv.createTemporaryView("DataSetTable", ds, $("x"), $("y"), $("z"));

	String sqlQuery = "SELECT x FROM DataSetTable WHERE z LIKE '%Hello%'";
	Table result = tableEnv.sqlQuery(sqlQuery);

	DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
	List<Row> results = resultSet.collect();
	String expected = "2\n" + "3\n" + "4";
	compareResultAsText(results, expected);
}
 
Example 4
Source File: GroupingTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
@SuppressWarnings("serial")
public void testGroupByKeySelector4() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	this.customTypeData.add(new CustomType());

	try {
		DataSet<CustomType> customDs = env.fromCollection(customTypeData);
		// should not work
		customDs.groupBy(
				new KeySelector<GroupingTest.CustomType, Tuple2<Integer, GroupingTest.CustomType>>() {
					@Override
					public Tuple2<Integer, CustomType> getKey(CustomType value) {
						return new Tuple2<Integer, CustomType>(value.myInt, value);
					}
				});
	} catch (Exception e) {
		Assert.fail();
	}
}
 
Example 5
Source File: JoinOperatorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test(expected = InvalidProgramException.class)
public void testJoinKeyMixing4() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<CustomType> ds2 = env.fromCollection(customTypeData);

	// should not work, more than one key field position
	ds1.join(ds2)
	.where(1, 3)
	.equalTo(
			new KeySelector<CustomType, Long>() {

					@Override
					public Long getKey(CustomType value) {
						return value.myLong;
					}
				}
			);
}
 
Example 6
Source File: GroupReduceITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testStringBasedDefinitionOnGroupSortForTwoGroupingKeys() throws Exception {
	/*
	 * Test string-based definition on group sort, for two grouping keys
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);

	DataSet<Tuple2<Tuple2<Integer, Integer>, String>> ds = CollectionDataSets.getGroupSortedNestedTupleDataSet(env);
	// f0.f0 is first integer
	DataSet<String> reduceDs = ds.groupBy("f1").sortGroup("f0.f0", Order.DESCENDING).sortGroup("f0.f1", Order.DESCENDING).reduceGroup(new NestedTupleReducer());
	List<String> result = reduceDs.collect();

	String expected = "a--(2,1)-(1,3)-(1,2)-\n" +
			"b--(2,2)-\n" +
			"c--(4,9)-(3,6)-(3,3)-\n";

	compareResultAsText(result, expected);
}
 
Example 7
Source File: CoGroupOperatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test(expected = InvalidProgramException.class)
public void testCoGroupKeyAtomicInvalidExpression4() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
	DataSet<Integer> ds2 = env.fromElements(0, 0, 1);

	ds1.coGroup(ds2).where("myInt").equalTo("*", "invalidKey");
}
 
Example 8
Source File: JoinWithEdgesITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testWithLessElements() throws Exception {
	/*
	 * Test joinWithEdges with the input DataSet passed as a parameter containing
	 * less elements than the edge DataSet, but of the same type
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	Graph<Long, Long, Long> res = graph.joinWithEdges(graph.getEdges().first(3)
		.map(new EdgeToTuple3Map<>()), new AddValuesMapper());

	DataSet<Edge<Long, Long>> data = res.getEdges();
	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "1,2,24\n" +
		"1,3,26\n" +
		"2,3,46\n" +
		"3,4,34\n" +
		"3,5,35\n" +
		"4,5,45\n" +
		"5,1,51\n";

	compareResultAsTuples(result, expectedResult);
}
 
Example 9
Source File: GroupCombineOperatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSemanticPropsWithKeySelector3() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);

	GroupCombineOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> combineOp =
			tupleDs.groupBy(new DummyTestKeySelector())
					.combineGroup(new DummyGroupCombineFunction2())
						.withForwardedFields("0->4;1;1->3;2");

	SemanticProperties semProps = combineOp.getSemanticProperties();

	assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
	assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
	assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
	assertTrue(semProps.getForwardingTargetFields(0, 2).contains(4));
	assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 2);
	assertTrue(semProps.getForwardingTargetFields(0, 3).contains(1));
	assertTrue(semProps.getForwardingTargetFields(0, 3).contains(3));
	assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
	assertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
	assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
	assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 0);

	assertTrue(semProps.getForwardingSourceField(0, 0) < 0);
	assertTrue(semProps.getForwardingSourceField(0, 1) == 3);
	assertTrue(semProps.getForwardingSourceField(0, 2) == 4);
	assertTrue(semProps.getForwardingSourceField(0, 3) == 3);
	assertTrue(semProps.getForwardingSourceField(0, 4) == 2);

	assertTrue(semProps.getReadFields(0).size() == 3);
	assertTrue(semProps.getReadFields(0).contains(2));
	assertTrue(semProps.getReadFields(0).contains(5));
	assertTrue(semProps.getReadFields(0).contains(6));
}
 
Example 10
Source File: FullOuterJoinOperatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void testFullOuterStrategies(JoinHint hint) {

		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
		DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);

		// should work
		ds1.fullOuterJoin(ds2, hint)
				.where(0).equalTo(4)
				.with(new DummyJoin());
	}
 
Example 11
Source File: GroupingTupleTranslationTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testCustomPartitioningTupleReduce() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		
		DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer, Integer>(0, 0))
				.rebalance().setParallelism(4);
		
		data.groupBy(0).withPartitioner(new TestPartitionerInt())
			.reduce(new SelectOneReducer<Tuple2<Integer,Integer>>())
			.output(new DiscardingOutputFormat<Tuple2<Integer, Integer>>());
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		SinkPlanNode sink = op.getDataSinks().iterator().next();
		SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
		SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
		
		assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
		assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
		assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 12
Source File: JoinITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testUDFJoinOnTuplesWithTupleReturningKeySelectors() throws Exception {
	/*
	 * UDF Join on tuples with tuple-returning key selectors
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
	DataSet<Tuple2<String, String>> joinDs =
			ds1.join(ds2)
					.where(new KeySelector3())
					.equalTo(new KeySelector4())
					.with(new T3T5FlatJoin());

	List<Tuple2<String, String>> result = joinDs.collect();

	String expected = "Hi,Hallo\n" +
			"Hello,Hallo Welt\n" +
			"Hello world,Hallo Welt wie gehts?\n" +
			"Hello world,ABC\n" +
			"I am fine.,HIJ\n" +
			"I am fine.,IJK\n";

	compareResultAsTuples(result, expected);
}
 
Example 13
Source File: BranchingPlansCompilerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testMultipleIterations() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(100);
	
	DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
	
	DataSet<String> reduced = input
			.map(new IdentityMapper<String>())
			.reduceGroup(new Top1GroupReducer<String>());
		
	IterativeDataSet<String> iteration1 = input.iterate(100);
	IterativeDataSet<String> iteration2 = input.iterate(20);
	IterativeDataSet<String> iteration3 = input.iterate(17);
	
	iteration1.closeWith(iteration1.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc1"))
			.output(new DiscardingOutputFormat<String>());
	iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>()).withBroadcastSet(reduced, "bc2"))
			.output(new DiscardingOutputFormat<String>());
	iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>()).withBroadcastSet(reduced, "bc3"))
			.output(new DiscardingOutputFormat<String>());
	
	Plan plan = env.createProgramPlan();
	
	try{
		compileNoStats(plan);
	}catch(Exception e){
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 14
Source File: PartitionOperatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testRebalance() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
	ds.rebalance();
}
 
Example 15
Source File: JoinWithEdgesITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testOnTargetWithNoCommonKeys() throws Exception {
	/*
	 * Test joinWithEdgesOnTarget with the input DataSet containing different keys than the edge DataSet
	 * - the iterator becomes empty.
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	Graph<Long, Long, Long> res = graph.joinWithEdgesOnTarget(TestGraphUtils.getLongLongTuple2TargetData(env),
		new DoubleValueMapper());

	DataSet<Edge<Long, Long>> data = res.getEdges();
	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "1,2,20\n" +
		"1,3,40\n" +
		"2,3,40\n" +
		"3,4,80\n" +
		"3,5,35\n" +
		"4,5,45\n" +
		"5,1,140\n";

	compareResultAsTuples(result, expectedResult);
}
 
Example 16
Source File: GroupCombineOperatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSemanticPropsWithKeySelector4() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);

	GroupCombineOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> combineOp =
			tupleDs.groupBy(new DummyTestKeySelector())
					.sortGroup(new DummyTestKeySelector(), Order.ASCENDING)
					.combineGroup(new DummyGroupCombineFunction2())
						.withForwardedFields("0->4;1;1->3;2");

	SemanticProperties semProps = combineOp.getSemanticProperties();

	assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
	assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
	assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 0);
	assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 0);
	assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
	assertTrue(semProps.getForwardingTargetFields(0, 4).contains(4));
	assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 2);
	assertTrue(semProps.getForwardingTargetFields(0, 5).contains(1));
	assertTrue(semProps.getForwardingTargetFields(0, 5).contains(3));
	assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 1);
	assertTrue(semProps.getForwardingTargetFields(0, 6).contains(2));
	assertTrue(semProps.getForwardingTargetFields(0, 7).size() == 0);
	assertTrue(semProps.getForwardingTargetFields(0, 8).size() == 0);

	assertTrue(semProps.getForwardingSourceField(0, 0) < 0);
	assertTrue(semProps.getForwardingSourceField(0, 1) == 5);
	assertTrue(semProps.getForwardingSourceField(0, 2) == 6);
	assertTrue(semProps.getForwardingSourceField(0, 3) == 5);
	assertTrue(semProps.getForwardingSourceField(0, 4) == 4);

	assertTrue(semProps.getReadFields(0).size() == 3);
	assertTrue(semProps.getReadFields(0).contains(4));
	assertTrue(semProps.getReadFields(0).contains(7));
	assertTrue(semProps.getReadFields(0).contains(8));
}
 
Example 17
Source File: GraphMutationsITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testAddEdgesInvalidVertices() throws Exception {
	/*
	 * Test addEdges() -- the source and target vertices do not exist in the graph
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
			TestGraphUtils.getLongLongEdgeData(env), env);

	List<Edge<Long, Long>> edgesToBeAdded = new ArrayList<>();
	edgesToBeAdded.add(new Edge<>(6L, 1L, 61L));
	edgesToBeAdded.add(new Edge<>(7L, 1L, 71L));

	graph = graph.addEdges(edgesToBeAdded);

	DataSet<Edge<Long, Long>> data = graph.getEdges();
	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "1,2,12\n" +
			"1,3,13\n" +
			"2,3,23\n" +
			"3,4,34\n" +
			"3,5,35\n" +
			"4,5,45\n" +
			"5,1,51\n";

	compareResultAsTuples(result, expectedResult);
}
 
Example 18
Source File: PregelSSSP.java    From flink with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {

		if (!parseParameters(args)) {
			return;
		}

		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

		DataSet<Edge<Long, Double>> edges = getEdgesDataSet(env);

		Graph<Long, Double, Double> graph = Graph.fromDataSet(edges, new InitVertices(), env);

		// Execute the vertex-centric iteration
		Graph<Long, Double, Double> result = graph.runVertexCentricIteration(
				new SSSPComputeFunction(srcVertexId), new SSSPCombiner(),
				maxIterations);

		// Extract the vertices as the result
		DataSet<Vertex<Long, Double>> singleSourceShortestPaths = result.getVertices();

		// emit result
		if (fileOutput) {
			singleSourceShortestPaths.writeAsCsv(outputPath, "\n", ",");
			env.execute("Pregel Single Source Shortest Paths Example");
		} else {
			singleSourceShortestPaths.print();
		}

	}
 
Example 19
Source File: JoinOperatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testJoinKeyExpressionsNested() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<NestedCustomType> ds1 = env.fromCollection(customNestedTypeData);
	DataSet<NestedCustomType> ds2 = env.fromCollection(customNestedTypeData);

	// should work
	try {
		ds1.join(ds2).where("myInt").equalTo("myInt");
	} catch (Exception e) {
		Assert.fail();
	}
}
 
Example 20
Source File: ClientTest.java    From flink with Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.fromElements(1, 2).output(new DiscardingOutputFormat<Integer>());
	env.execute().getJobID();
}