org.apache.flink.api.common.operators.base.GroupReduceOperatorBase Java Examples

The following examples show how to use org.apache.flink.api.common.operators.base.GroupReduceOperatorBase. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AggregateTranslationTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void translateAggregate() {
	try {
		final int parallelism = 8;
		ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(parallelism);

		@SuppressWarnings("unchecked")
		DataSet<Tuple3<Double, StringValue, Long>> initialData =
				env.fromElements(new Tuple3<Double, StringValue, Long>(3.141592, new StringValue("foobar"), Long.valueOf(77)));

		initialData.groupBy(0).aggregate(Aggregations.MIN, 1).and(Aggregations.SUM, 2).output(new DiscardingOutputFormat<Tuple3<Double, StringValue, Long>>());

		Plan p = env.createProgramPlan();

		GenericDataSinkBase<?> sink = p.getDataSinks().iterator().next();

		GroupReduceOperatorBase<?, ?, ?> reducer = (GroupReduceOperatorBase<?, ?, ?>) sink.getInput();

		// check keys
		assertEquals(1, reducer.getKeyColumns(0).length);
		assertEquals(0, reducer.getKeyColumns(0)[0]);

		assertEquals(-1, reducer.getParallelism());
		assertTrue(reducer.isCombinable());

		assertTrue(reducer.getInput() instanceof GenericDataSourceBase<?, ?>);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		fail("Test caused an error: " + e.getMessage());
	}
}
Example #2
Source File: PartialGroupProperties.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	// create in input node for combine with the same parallelism as input node
	GroupReduceNode combinerNode = new GroupReduceNode((GroupReduceOperatorBase<?, ?, ?>) node.getOperator());
	combinerNode.setParallelism(in.getSource().getParallelism());

	SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getOperator().getName()+")", in,
			DriverStrategy.SORTED_GROUP_COMBINE);
	// sorting key info
	combiner.setDriverKeyInfo(in.getLocalStrategyKeys(), in.getLocalStrategySortOrder(), 0);
	// set grouping comparator key info
	combiner.setDriverKeyInfo(this.keyList, 1);
	
	return combiner;
}
Example #3
Source File: GroupReduceNode.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a new optimizer node for the given operator.
 * 
 * @param operator The reduce operation.
 */
public GroupReduceNode(GroupReduceOperatorBase<?, ?, ?> operator) {
	super(operator);
	this.operatorName = "GroupReduce";
	
	if (this.keys == null) {
		// case of a key-less reducer. force a parallelism of 1
		setParallelism(1);
	}
	
	this.possibleProperties = initPossibleProperties(operator.getCustomPartitioner());
}
Example #4
Source File: GroupReduceNodeTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testGetSemanticProperties() {

	SingleInputSemanticProperties origProps = new SingleInputSemanticProperties();
	origProps.addForwardedField(0, 1);
	origProps.addForwardedField(2, 2);
	origProps.addForwardedField(3, 4);
	origProps.addForwardedField(6, 0);
	origProps.addReadFields(new FieldSet(0, 2, 4, 7));

	GroupReduceOperatorBase<?,?,?> op = mock(GroupReduceOperatorBase.class);
	when(op.getSemanticProperties()).thenReturn(origProps);
	when(op.getKeyColumns(0)).thenReturn(new int[]{3,2});
	when(op.getParameters()).thenReturn(new Configuration());

	GroupReduceNode node = new GroupReduceNode(op);

	SemanticProperties filteredProps = node.getSemanticPropertiesForLocalPropertyFiltering();

	assertTrue(filteredProps.getForwardingTargetFields(0, 0).size() == 0);
	assertTrue(filteredProps.getForwardingTargetFields(0, 2).size() == 1);
	assertTrue(filteredProps.getForwardingTargetFields(0, 2).contains(2));
	assertTrue(filteredProps.getForwardingTargetFields(0, 3).size() == 1);
	assertTrue(filteredProps.getForwardingTargetFields(0, 3).contains(4));
	assertTrue(filteredProps.getForwardingTargetFields(0, 6).size() == 0);
	assertTrue(filteredProps.getForwardingSourceField(0, 1) < 0);
	assertTrue(filteredProps.getForwardingSourceField(0, 2) == 2);
	assertTrue(filteredProps.getForwardingSourceField(0, 4) == 3);
	assertTrue(filteredProps.getForwardingSourceField(0, 0) < 0);

	assertTrue(filteredProps.getReadFields(0).size() == 4);
	assertTrue(filteredProps.getReadFields(0).contains(0));
	assertTrue(filteredProps.getReadFields(0).contains(2));
	assertTrue(filteredProps.getReadFields(0).contains(4));
	assertTrue(filteredProps.getReadFields(0).contains(7));

}
Example #5
Source File: AggregateTranslationTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void translateAggregate() {
	try {
		final int parallelism = 8;
		ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(parallelism);

		@SuppressWarnings("unchecked")
		DataSet<Tuple3<Double, StringValue, Long>> initialData =
				env.fromElements(new Tuple3<Double, StringValue, Long>(3.141592, new StringValue("foobar"), Long.valueOf(77)));

		initialData.groupBy(0).aggregate(Aggregations.MIN, 1).and(Aggregations.SUM, 2).output(new DiscardingOutputFormat<Tuple3<Double, StringValue, Long>>());

		Plan p = env.createProgramPlan();

		GenericDataSinkBase<?> sink = p.getDataSinks().iterator().next();

		GroupReduceOperatorBase<?, ?, ?> reducer = (GroupReduceOperatorBase<?, ?, ?>) sink.getInput();

		// check keys
		assertEquals(1, reducer.getKeyColumns(0).length);
		assertEquals(0, reducer.getKeyColumns(0)[0]);

		assertEquals(-1, reducer.getParallelism());
		assertTrue(reducer.isCombinable());

		assertTrue(reducer.getInput() instanceof GenericDataSourceBase<?, ?>);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		fail("Test caused an error: " + e.getMessage());
	}
}
Example #6
Source File: PartialGroupProperties.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	// create in input node for combine with the same parallelism as input node
	GroupReduceNode combinerNode = new GroupReduceNode((GroupReduceOperatorBase<?, ?, ?>) node.getOperator());
	combinerNode.setParallelism(in.getSource().getParallelism());

	SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getOperator().getName()+")", in,
			DriverStrategy.SORTED_GROUP_COMBINE);
	// sorting key info
	combiner.setDriverKeyInfo(in.getLocalStrategyKeys(), in.getLocalStrategySortOrder(), 0);
	// set grouping comparator key info
	combiner.setDriverKeyInfo(this.keyList, 1);
	
	return combiner;
}
Example #7
Source File: GroupReduceNode.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a new optimizer node for the given operator.
 * 
 * @param operator The reduce operation.
 */
public GroupReduceNode(GroupReduceOperatorBase<?, ?, ?> operator) {
	super(operator);
	this.operatorName = "GroupReduce";
	
	if (this.keys == null) {
		// case of a key-less reducer. force a parallelism of 1
		setParallelism(1);
	}
	
	this.possibleProperties = initPossibleProperties(operator.getCustomPartitioner());
}
Example #8
Source File: GroupReduceNodeTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testGetSemanticProperties() {

	SingleInputSemanticProperties origProps = new SingleInputSemanticProperties();
	origProps.addForwardedField(0, 1);
	origProps.addForwardedField(2, 2);
	origProps.addForwardedField(3, 4);
	origProps.addForwardedField(6, 0);
	origProps.addReadFields(new FieldSet(0, 2, 4, 7));

	GroupReduceOperatorBase<?,?,?> op = mock(GroupReduceOperatorBase.class);
	when(op.getSemanticProperties()).thenReturn(origProps);
	when(op.getKeyColumns(0)).thenReturn(new int[]{3,2});
	when(op.getParameters()).thenReturn(new Configuration());

	GroupReduceNode node = new GroupReduceNode(op);

	SemanticProperties filteredProps = node.getSemanticPropertiesForLocalPropertyFiltering();

	assertTrue(filteredProps.getForwardingTargetFields(0, 0).size() == 0);
	assertTrue(filteredProps.getForwardingTargetFields(0, 2).size() == 1);
	assertTrue(filteredProps.getForwardingTargetFields(0, 2).contains(2));
	assertTrue(filteredProps.getForwardingTargetFields(0, 3).size() == 1);
	assertTrue(filteredProps.getForwardingTargetFields(0, 3).contains(4));
	assertTrue(filteredProps.getForwardingTargetFields(0, 6).size() == 0);
	assertTrue(filteredProps.getForwardingSourceField(0, 1) < 0);
	assertTrue(filteredProps.getForwardingSourceField(0, 2) == 2);
	assertTrue(filteredProps.getForwardingSourceField(0, 4) == 3);
	assertTrue(filteredProps.getForwardingSourceField(0, 0) < 0);

	assertTrue(filteredProps.getReadFields(0).size() == 4);
	assertTrue(filteredProps.getReadFields(0).contains(0));
	assertTrue(filteredProps.getReadFields(0).contains(2));
	assertTrue(filteredProps.getReadFields(0).contains(4));
	assertTrue(filteredProps.getReadFields(0).contains(7));

}
Example #9
Source File: AggregateTranslationTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void translateAggregate() {
	try {
		final int parallelism = 8;
		ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(parallelism);

		@SuppressWarnings("unchecked")
		DataSet<Tuple3<Double, StringValue, Long>> initialData =
				env.fromElements(new Tuple3<Double, StringValue, Long>(3.141592, new StringValue("foobar"), Long.valueOf(77)));

		initialData.groupBy(0).aggregate(Aggregations.MIN, 1).and(Aggregations.SUM, 2).output(new DiscardingOutputFormat<Tuple3<Double, StringValue, Long>>());

		Plan p = env.createProgramPlan();

		GenericDataSinkBase<?> sink = p.getDataSinks().iterator().next();

		GroupReduceOperatorBase<?, ?, ?> reducer = (GroupReduceOperatorBase<?, ?, ?>) sink.getInput();

		// check keys
		assertEquals(1, reducer.getKeyColumns(0).length);
		assertEquals(0, reducer.getKeyColumns(0)[0]);

		assertEquals(-1, reducer.getParallelism());
		assertTrue(reducer.isCombinable());

		assertTrue(reducer.getInput() instanceof GenericDataSourceBase<?, ?>);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		fail("Test caused an error: " + e.getMessage());
	}
}
Example #10
Source File: PartialGroupProperties.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	// create in input node for combine with the same parallelism as input node
	GroupReduceNode combinerNode = new GroupReduceNode((GroupReduceOperatorBase<?, ?, ?>) node.getOperator());
	combinerNode.setParallelism(in.getSource().getParallelism());

	SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getOperator().getName()+")", in,
			DriverStrategy.SORTED_GROUP_COMBINE);
	// sorting key info
	combiner.setDriverKeyInfo(in.getLocalStrategyKeys(), in.getLocalStrategySortOrder(), 0);
	// set grouping comparator key info
	combiner.setDriverKeyInfo(this.keyList, 1);
	
	return combiner;
}
Example #11
Source File: GroupReduceNode.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a new optimizer node for the given operator.
 * 
 * @param operator The reduce operation.
 */
public GroupReduceNode(GroupReduceOperatorBase<?, ?, ?> operator) {
	super(operator);
	this.operatorName = "GroupReduce";
	
	if (this.keys == null) {
		// case of a key-less reducer. force a parallelism of 1
		setParallelism(1);
	}
	
	this.possibleProperties = initPossibleProperties(operator.getCustomPartitioner());
}
Example #12
Source File: GroupReduceNodeTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testGetSemanticProperties() {

	SingleInputSemanticProperties origProps = new SingleInputSemanticProperties();
	origProps.addForwardedField(0, 1);
	origProps.addForwardedField(2, 2);
	origProps.addForwardedField(3, 4);
	origProps.addForwardedField(6, 0);
	origProps.addReadFields(new FieldSet(0, 2, 4, 7));

	GroupReduceOperatorBase<?,?,?> op = mock(GroupReduceOperatorBase.class);
	when(op.getSemanticProperties()).thenReturn(origProps);
	when(op.getKeyColumns(0)).thenReturn(new int[]{3,2});
	when(op.getParameters()).thenReturn(new Configuration());

	GroupReduceNode node = new GroupReduceNode(op);

	SemanticProperties filteredProps = node.getSemanticPropertiesForLocalPropertyFiltering();

	assertTrue(filteredProps.getForwardingTargetFields(0, 0).size() == 0);
	assertTrue(filteredProps.getForwardingTargetFields(0, 2).size() == 1);
	assertTrue(filteredProps.getForwardingTargetFields(0, 2).contains(2));
	assertTrue(filteredProps.getForwardingTargetFields(0, 3).size() == 1);
	assertTrue(filteredProps.getForwardingTargetFields(0, 3).contains(4));
	assertTrue(filteredProps.getForwardingTargetFields(0, 6).size() == 0);
	assertTrue(filteredProps.getForwardingSourceField(0, 1) < 0);
	assertTrue(filteredProps.getForwardingSourceField(0, 2) == 2);
	assertTrue(filteredProps.getForwardingSourceField(0, 4) == 3);
	assertTrue(filteredProps.getForwardingSourceField(0, 0) < 0);

	assertTrue(filteredProps.getReadFields(0).size() == 4);
	assertTrue(filteredProps.getReadFields(0).contains(0));
	assertTrue(filteredProps.getReadFields(0).contains(2));
	assertTrue(filteredProps.getReadFields(0).contains(4));
	assertTrue(filteredProps.getReadFields(0).contains(7));

}
Example #13
Source File: GroupReduceNode.java    From Flink-CEPplus with Apache License 2.0 2 votes vote down vote up 
/**
 * Gets the operator represented by this optimizer node.
 * 
 * @return The operator represented by this optimizer node.
 */
@Override
public GroupReduceOperatorBase<?, ?, ?> getOperator() {
	return (GroupReduceOperatorBase<?, ?, ?>) super.getOperator();
}
Example #14
Source File: GroupReduceNode.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Gets the operator represented by this optimizer node.
 * 
 * @return The operator represented by this optimizer node.
 */
@Override
public GroupReduceOperatorBase<?, ?, ?> getOperator() {
	return (GroupReduceOperatorBase<?, ?, ?>) super.getOperator();
}
Example #15
Source File: GroupReduceNode.java    From flink with Apache License 2.0 2 votes vote down vote up 
/**
 * Gets the operator represented by this optimizer node.
 * 
 * @return The operator represented by this optimizer node.
 */
@Override
public GroupReduceOperatorBase<?, ?, ?> getOperator() {
	return (GroupReduceOperatorBase<?, ?, ?>) super.getOperator();
}