org.apache.flink.api.common.InvalidProgramException Java Examples

/**
 * Intermediate step of a CoGroup transformation.
 *
 * <p>To continue the CoGroup transformation, provide a {@link org.apache.flink.api.common.functions.RichCoGroupFunction} by calling
 * {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate.CoGroupOperatorWithoutFunction#with(org.apache.flink.api.common.functions.CoGroupFunction)}.
 *
 */
private CoGroupOperatorWithoutFunction createCoGroupOperator(Keys<I2> keys2) {
	if (keys2 == null) {
		throw new NullPointerException();
	}

	if (keys2.isEmpty()) {
		throw new InvalidProgramException("The co-group keys must not be empty.");
	}
	try {
		keys1.areCompatible(keys2);
	} catch (IncompatibleKeysException ike) {
		throw new InvalidProgramException("The pair of co-group keys are not compatible with each other.", ike);
	}

	return new CoGroupOperatorWithoutFunction(keys2);
}
 

public static boolean isSortKey(int fieldPos, TypeInformation<?> type) {

			if (!type.isTupleType() || !(type instanceof CompositeType)) {
				throw new InvalidProgramException("Specifying keys via field positions is only valid " +
					"for tuple data types. Type: " + type);
			}
			if (type.getArity() == 0) {
				throw new InvalidProgramException("Tuple size must be greater than 0. Size: " + type.getArity());
			}

			if(fieldPos < 0 || fieldPos >= type.getArity()) {
				throw new IndexOutOfBoundsException("Tuple position is out of range: " + fieldPos);
			}

			TypeInformation<?> sortKeyType = ((CompositeType<?>)type).getTypeAt(fieldPos);
			return sortKeyType.isSortKeyType();
		}
 

@Test(expected = InvalidProgramException.class)
public void testJoinKeyMixing4() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<CustomType> ds2 = env.fromCollection(customTypeData);

	// should not work, more than one key field position
	ds1.join(ds2)
	.where(1, 3)
	.equalTo(
			new KeySelector<CustomType, Long>() {

					@Override
					public Long getKey(CustomType value) {
						return value.myLong;
					}
				}
			);
}
 

@Test(expected = InvalidProgramException.class)
@SuppressWarnings("serial")
public void testGroupByKeySelector5() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	this.customTypeData.add(new CustomType());

	DataSet<CustomType> customDs = env.fromCollection(customTypeData);
	// should not work
	customDs.groupBy(
			new KeySelector<GroupingTest.CustomType, CustomType2>() {
				@Override
				public CustomType2 getKey(CustomType value) {
					return new CustomType2();
				}
			});
}
 

@Test
public void testPOJOWithNestedArrayNoHashCodeKeyRejection() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<POJOWithHashCode> input = env.fromElements(
			new POJOWithHashCode(new int[] {1, 2}));

	TypeInformation<?> expectedTypeInfo = new TupleTypeInfo<Tuple1<int[]>>(
			PrimitiveArrayTypeInfo.INT_PRIMITIVE_ARRAY_TYPE_INFO);

	// adjust the rule
	expectedException.expect(InvalidProgramException.class);
	expectedException.expectMessage(new StringStartsWith("Type " + expectedTypeInfo + " cannot be used as key."));

	input.keyBy("id");
}
 

/**
 * Grouped aggregation.
 *
 * @param input
 * @param function
 * @param field
 */
public AggregateOperator(Grouping<IN> input, Aggregations function, int field, String aggregateLocationName) {
	super(Preconditions.checkNotNull(input).getInputDataSet(), input.getInputDataSet().getType());
	Preconditions.checkNotNull(function);

	this.aggregateLocationName = aggregateLocationName;

	if (!input.getInputDataSet().getType().isTupleType()) {
		throw new InvalidProgramException("Aggregating on field positions is only possible on tuple data types.");
	}

	TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) input.getInputDataSet().getType();

	if (field < 0 || field >= inType.getArity()) {
		throw new IllegalArgumentException("Aggregation field position is out of range.");
	}

	AggregationFunctionFactory factory = function.getFactory();
	AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());

	// set the aggregation fields
	this.aggregationFunctions.add(aggFunct);
	this.fields.add(field);
	this.grouping = input;
}
 

@Test(expected = InvalidProgramException.class)
public void testJoinKeyMixing3() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<CustomType> ds2 = env.fromCollection(customTypeData);

	// should not work, incompatible types
	ds1.join(ds2)
	.where(2)
	.equalTo(
			new KeySelector<CustomType, Long>() {

					@Override
					public Long getKey(CustomType value) {
						return value.myLong;
					}
				}
			);
}
 

/**
 * Defines that data is partitioned using a specific partitioning method
 * across input splits on the fields defined by field positions.
 * All records sharing the same key (combination) must be contained in a single input split.
 *
 * <p><b>
 *     IMPORTANT: Providing wrong information with SplitDataProperties can cause wrong results!
 * </b>
 *
 * @param partitionMethodId An ID for the method that was used to partition the data across splits.
 * @param partitionFields The field positions of the partitioning keys.
 * @return This SplitDataProperties object.
 */
public SplitDataProperties<T> splitsPartitionedBy(String partitionMethodId, int... partitionFields) {

	if (partitionFields == null) {
		throw new InvalidProgramException("PartitionFields may not be null.");
	} else if (partitionFields.length == 0) {
		throw new InvalidProgramException("PartitionFields may not be empty.");
	}

	this.splitPartitionKeys = getAllFlatKeys(partitionFields);
	if (partitionMethodId != null) {
		this.splitPartitioner = new SourcePartitionerMarker<>(partitionMethodId);
	} else {
		this.splitPartitioner = null;
	}

	return this;
}
 

@Test(expected = InvalidProgramException.class)
public void testJoinKeyMixing3() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<CustomType> ds2 = env.fromCollection(customTypeData);

	// should not work, incompatible types
	ds1.join(ds2)
	.where(2)
	.equalTo(
			new KeySelector<CustomType, Long>() {

					@Override
					public Long getKey(CustomType value) {
						return value.myLong;
					}
				}
			);
}
 

@Test(expected = InvalidProgramException.class)
public void testGroupSortKeyFields3() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Long> longDs = env.fromCollection(emptyLongData, BasicTypeInfo.LONG_TYPE_INFO);

	// should not work: sorted groups on groupings by key selectors
	longDs.groupBy(new KeySelector<Long, Long>() {
		private static final long serialVersionUID = 1L;

		@Override
		public Long getKey(Long value) {
			return value;
		}

	}).sortGroup(0, Order.ASCENDING);

}
 

/**
 * Defines that data is partitioned using a specific partitioning method
 * across input splits on the fields defined by field positions.
 * All records sharing the same key (combination) must be contained in a single input split.
 *
 * <p><b>
 *     IMPORTANT: Providing wrong information with SplitDataProperties can cause wrong results!
 * </b>
 *
 * @param partitionMethodId An ID for the method that was used to partition the data across splits.
 * @param partitionFields The field positions of the partitioning keys.
 * @return This SplitDataProperties object.
 */
public SplitDataProperties<T> splitsPartitionedBy(String partitionMethodId, int... partitionFields) {

	if (partitionFields == null) {
		throw new InvalidProgramException("PartitionFields may not be null.");
	} else if (partitionFields.length == 0) {
		throw new InvalidProgramException("PartitionFields may not be empty.");
	}

	this.splitPartitionKeys = getAllFlatKeys(partitionFields);
	if (partitionMethodId != null) {
		this.splitPartitioner = new SourcePartitionerMarker<>(partitionMethodId);
	} else {
		this.splitPartitioner = null;
	}

	return this;
}
 

/**
 * Defines that the data within an input split is grouped on the fields defined by the field positions.
 * All records sharing the same key (combination) must be subsequently emitted by the input
 * format for each input split.
 *
 * <p><b>
 *     IMPORTANT: Providing wrong information with SplitDataProperties can cause wrong results!
 * </b>
 *
 * @param groupFields The field positions of the grouping keys.
 * @return This SplitDataProperties object.
 */
public SplitDataProperties<T> splitsGroupedBy(int... groupFields) {

	if (groupFields == null) {
		throw new InvalidProgramException("GroupFields may not be null.");
	} else if (groupFields.length == 0) {
		throw new InvalidProgramException("GroupFields may not be empty.");
	}

	if (this.splitOrdering != null) {
		throw new InvalidProgramException("DataSource may either be grouped or sorted.");
	}

	this.splitGroupKeys = getAllFlatKeys(groupFields);

	return this;
}
 

@Test(expected = InvalidProgramException.class)
public void testCoGroupKeyMixing4() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<CustomType> ds2 = env.fromCollection(customTypeData);

	// should not work, more than one key field position
	ds1.coGroup(ds2)
	.where(1, 3)
	.equalTo(
			new KeySelector<CustomType, Long>() {

					@Override
					public Long getKey(CustomType value) {
						return value.myLong;
					}
				}
			);
}
 

@Test(expected = InvalidProgramException.class)
public void testCoGroupKeyAtomicInvalidExpression4() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
	DataSet<Integer> ds2 = env.fromElements(0, 0, 1);

	ds1.coGroup(ds2).where("myInt").equalTo("*", "invalidKey");
}
 

@Test(expected = InvalidProgramException.class)
public void testGroupByKeyFields2() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Long> longDs = env.fromCollection(emptyLongData, BasicTypeInfo.LONG_TYPE_INFO);
	// should not work: groups on basic type
	longDs.groupBy(0);
}
 

@Override
public void postVisit(PlanNode node) {

	if(node instanceof IterationPlanNode) {
		IterationPlanNode iNode = (IterationPlanNode)node;
		if(!visitedIterationNodes.contains(iNode)) {
			visitedIterationNodes.add(iNode);
			iNode.acceptForStepFunction(this);
		}
	}

	final Iterable<Channel> inputChannels = node.getInputs();
	for (Channel channel : inputChannels) {
		ShipStrategyType shipStrategy = channel.getShipStrategy();
		// Make sure we only optimize the DAG for range partition, and do not optimize multi times.
		if (shipStrategy == ShipStrategyType.PARTITION_RANGE) {

			if(channel.getDataDistribution() == null) {
				if (node.isOnDynamicPath()) {
					throw new InvalidProgramException("Range Partitioning not supported within iterations if users do not supply the data distribution.");
				}

				PlanNode channelSource = channel.getSource();
				List<Channel> newSourceOutputChannels = rewriteRangePartitionChannel(channel);
				channelSource.getOutgoingChannels().remove(channel);
				channelSource.getOutgoingChannels().addAll(newSourceOutputChannels);
			}
		}
	}
}
 

@Test(expected = InvalidProgramException.class)
public void testCoGroupKeyAtomicInvalidExpression5() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<ArrayList<Integer>> ds1 = env.fromElements(new ArrayList<Integer>());
	DataSet<Integer> ds2 = env.fromElements(0, 0, 0);

	ds1.coGroup(ds2).where("*");
}
 

public void setDetachedPlan(FlinkPlan plan) {
	if (detachedPlan == null) {
		detachedPlan = plan;
	} else {
		throw new InvalidProgramException(DetachedJobExecutionResult.DETACHED_MESSAGE +
				DetachedJobExecutionResult.EXECUTE_TWICE_MESSAGE);
	}
}
 

public static void ensureSerializable(Object obj) {
	try {
		InstantiationUtil.serializeObject(obj);
	} catch (Exception e) {
		throw new InvalidProgramException("Object " + obj + " is not serializable", e);
	}
}
 

@Test(expected = InvalidProgramException.class)
public void testFullOuter9() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);

	// key types do not match
	ds1.fullOuterJoin(ds2)
			.where(0).equalTo(1)
			.with(new DummyJoin());
}
 

private int[] getAllFlatKeys(String[] fieldExpressions) {

		int[] allKeys = null;

		for (String keyExp : fieldExpressions) {
			Keys.ExpressionKeys<T> ek = new Keys.ExpressionKeys<>(keyExp, this.type);
			int[] flatKeys = ek.computeLogicalKeyPositions();

			if (allKeys == null) {
				allKeys = flatKeys;
			} else {
				// check for duplicates
				for (int key1 : flatKeys) {
					for (int key2 : allKeys) {
						if (key1 == key2) {
							throw new InvalidProgramException("Duplicate fields in field expression " + keyExp);
						}
					}
				}
				// append flat keys
				int oldLength = allKeys.length;
				int newLength = oldLength + flatKeys.length;
				allKeys = Arrays.copyOf(allKeys, newLength);
				System.arraycopy(flatKeys, 0, allKeys, oldLength, flatKeys.length);
			}
		}

		return allKeys;
	}
 

@Test(expected = InvalidProgramException.class)
public void testDistinctOnNotKeyDataType() throws Exception {
   	/*
    	* should not work. NotComparable data type cannot be used as key
    	*/
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	NotComparable a = new NotComparable();
	List<NotComparable> l = new ArrayList<NotComparable>();
	l.add(a);

	DataSet<NotComparable> ds = env.fromCollection(l);
	DataSet<NotComparable> reduceDs = ds.distinct();

}
 

/**
 * Returns a new set containing the first n elements in this grouped and sorted {@link DataSet}.
 * @param n The desired number of elements for each group.
 * @return A GroupReduceOperator that represents the DataSet containing the elements.
*/
public GroupReduceOperator<T, T> first(int n) {
	if (n < 1) {
		throw new InvalidProgramException("Parameter n of first(n) must be at least 1.");
	}

	return reduceGroup(new FirstReducer<T>(n));
}
 

@Test(expected = InvalidProgramException.class)
public void testSortPartitionWithExpressionKeys4() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs = env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);

	// must not work
	tupleDs.sortPartition("f3", Order.ASCENDING);
}
 

@Test(expected = InvalidProgramException.class)
public void testRightOuter9() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);

	// key types do not match
	ds1.rightOuterJoin(ds2)
			.where(0).equalTo(1)
			.with(new DummyJoin());
}
 

@Test(expected = InvalidProgramException.class)
public void testFailPrimitiveOrder3() {

	final ExecutionEnvironment env = ExecutionEnvironment
			.getExecutionEnvironment();
	DataSet<Long> longDs = env
			.generateSequence(0, 2);

	// must not work
	longDs.writeAsText("/tmp/willNotHappen")
		.sortLocalOutput("nope", Order.ASCENDING);
}
 

@Test(expected = InvalidProgramException.class)
public void testSortPartitionWithPositionKeys3() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs = env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);

	// must not work
	tupleDs.sortPartition(2, Order.ASCENDING);
}
 

@Test(expected = InvalidProgramException.class)
public void testJoinKeyMixedTupleIndexWrongType() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	ds1.join(ds2).where("f0").equalTo(3); // 3 is of type long, so it should fail
}
 

/**
 * Creates a new mini cluster stream environment that configures its local executor with the given configuration.
 *
 * @param configuration The configuration used to configure the local executor.
 */
public LocalStreamEnvironment(@Nonnull Configuration configuration) {
	if (!ExecutionEnvironment.areExplicitEnvironmentsAllowed()) {
		throw new InvalidProgramException(
			"The LocalStreamEnvironment cannot be used when submitting a program through a client, " +
				"or running in a TestEnvironment context.");
	}
	this.configuration = configuration;
	setParallelism(1);
}
 

public SortedGrouping(DataSet<T> set, Keys<T> keys, int field, Order order) {
	super(set, keys);

	if (!Keys.ExpressionKeys.isSortKey(field, inputDataSet.getType())) {
		throw new InvalidProgramException("Selected sort key is not a sortable type");
	}

	// use int-based expression key to properly resolve nested tuples for grouping
	ExpressionKeys<T> ek = new ExpressionKeys<>(field, inputDataSet.getType());

	this.groupSortKeyPositions = ek.computeLogicalKeyPositions();
	this.groupSortOrders = new Order[groupSortKeyPositions.length];
	Arrays.fill(this.groupSortOrders, order);
}