Java Code Examples for org.apache.flink.api.common.InvalidProgramException

@Test(expected = InvalidProgramException.class)
@SuppressWarnings("serial")
public void testGroupByKeySelector5() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	this.customTypeData.add(new CustomType());

	DataSet<CustomType> customDs = env.fromCollection(customTypeData);
	// should not work
	customDs.groupBy(
			new KeySelector<GroupingTest.CustomType, CustomType2>() {
				@Override
				public CustomType2 getKey(CustomType value) {
					return new CustomType2();
				}
			});
}
 

@Test(expected = InvalidProgramException.class)
public void testJoinKeyMixing4() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<CustomType> ds2 = env.fromCollection(customTypeData);

	// should not work, more than one key field position
	ds1.join(ds2)
	.where(1, 3)
	.equalTo(
			new KeySelector<CustomType, Long>() {

					@Override
					public Long getKey(CustomType value) {
						return value.myLong;
					}
				}
			);
}
 

public static boolean isSortKey(int fieldPos, TypeInformation<?> type) {

			if (!type.isTupleType() || !(type instanceof CompositeType)) {
				throw new InvalidProgramException("Specifying keys via field positions is only valid " +
					"for tuple data types. Type: " + type);
			}
			if (type.getArity() == 0) {
				throw new InvalidProgramException("Tuple size must be greater than 0. Size: " + type.getArity());
			}

			if(fieldPos < 0 || fieldPos >= type.getArity()) {
				throw new IndexOutOfBoundsException("Tuple position is out of range: " + fieldPos);
			}

			TypeInformation<?> sortKeyType = ((CompositeType<?>)type).getTypeAt(fieldPos);
			return sortKeyType.isSortKeyType();
		}
 

/**
 * Intermediate step of a CoGroup transformation.
 *
 * <p>To continue the CoGroup transformation, provide a {@link org.apache.flink.api.common.functions.RichCoGroupFunction} by calling
 * {@link org.apache.flink.api.java.operators.CoGroupOperator.CoGroupOperatorSets.CoGroupOperatorSetsPredicate.CoGroupOperatorWithoutFunction#with(org.apache.flink.api.common.functions.CoGroupFunction)}.
 *
 */
private CoGroupOperatorWithoutFunction createCoGroupOperator(Keys<I2> keys2) {
	if (keys2 == null) {
		throw new NullPointerException();
	}

	if (keys2.isEmpty()) {
		throw new InvalidProgramException("The co-group keys must not be empty.");
	}
	try {
		keys1.areCompatible(keys2);
	} catch (IncompatibleKeysException ike) {
		throw new InvalidProgramException("The pair of co-group keys are not compatible with each other.", ike);
	}

	return new CoGroupOperatorWithoutFunction(keys2);
}
 

/**
 * Grouped aggregation.
 *
 * @param input
 * @param function
 * @param field
 */
public AggregateOperator(Grouping<IN> input, Aggregations function, int field, String aggregateLocationName) {
	super(Preconditions.checkNotNull(input).getInputDataSet(), input.getInputDataSet().getType());
	Preconditions.checkNotNull(function);

	this.aggregateLocationName = aggregateLocationName;

	if (!input.getInputDataSet().getType().isTupleType()) {
		throw new InvalidProgramException("Aggregating on field positions is only possible on tuple data types.");
	}

	TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) input.getInputDataSet().getType();

	if (field < 0 || field >= inType.getArity()) {
		throw new IllegalArgumentException("Aggregation field position is out of range.");
	}

	AggregationFunctionFactory factory = function.getFactory();
	AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());

	// set the aggregation fields
	this.aggregationFunctions.add(aggFunct);
	this.fields.add(field);
	this.grouping = input;
}
 

@Test(expected = InvalidProgramException.class)
public void testJoinKeyMixing3() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<CustomType> ds2 = env.fromCollection(customTypeData);

	// should not work, incompatible types
	ds1.join(ds2)
	.where(2)
	.equalTo(
			new KeySelector<CustomType, Long>() {

					@Override
					public Long getKey(CustomType value) {
						return value.myLong;
					}
				}
			);
}
 

@Test(expected = InvalidProgramException.class)
public void testGroupSortKeyFields3() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Long> longDs = env.fromCollection(emptyLongData, BasicTypeInfo.LONG_TYPE_INFO);

	// should not work: sorted groups on groupings by key selectors
	longDs.groupBy(new KeySelector<Long, Long>() {
		private static final long serialVersionUID = 1L;

		@Override
		public Long getKey(Long value) {
			return value;
		}

	}).sortGroup(0, Order.ASCENDING);

}
 

/**
 * Defines that data is partitioned using a specific partitioning method
 * across input splits on the fields defined by field positions.
 * All records sharing the same key (combination) must be contained in a single input split.
 *
 * <p><b>
 *     IMPORTANT: Providing wrong information with SplitDataProperties can cause wrong results!
 * </b>
 *
 * @param partitionMethodId An ID for the method that was used to partition the data across splits.
 * @param partitionFields The field positions of the partitioning keys.
 * @return This SplitDataProperties object.
 */
public SplitDataProperties<T> splitsPartitionedBy(String partitionMethodId, int... partitionFields) {

	if (partitionFields == null) {
		throw new InvalidProgramException("PartitionFields may not be null.");
	} else if (partitionFields.length == 0) {
		throw new InvalidProgramException("PartitionFields may not be empty.");
	}

	this.splitPartitionKeys = getAllFlatKeys(partitionFields);
	if (partitionMethodId != null) {
		this.splitPartitioner = new SourcePartitionerMarker<>(partitionMethodId);
	} else {
		this.splitPartitioner = null;
	}

	return this;
}
 

/**
 * Defines that the data within an input split is grouped on the fields defined by the field positions.
 * All records sharing the same key (combination) must be subsequently emitted by the input
 * format for each input split.
 *
 * <p><b>
 *     IMPORTANT: Providing wrong information with SplitDataProperties can cause wrong results!
 * </b>
 *
 * @param groupFields The field positions of the grouping keys.
 * @return This SplitDataProperties object.
 */
public SplitDataProperties<T> splitsGroupedBy(int... groupFields) {

	if (groupFields == null) {
		throw new InvalidProgramException("GroupFields may not be null.");
	} else if (groupFields.length == 0) {
		throw new InvalidProgramException("GroupFields may not be empty.");
	}

	if (this.splitOrdering != null) {
		throw new InvalidProgramException("DataSource may either be grouped or sorted.");
	}

	this.splitGroupKeys = getAllFlatKeys(groupFields);

	return this;
}
 

@Test(expected = InvalidProgramException.class)
public void testCoGroupKeyMixing4() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<CustomType> ds2 = env.fromCollection(customTypeData);

	// should not work, more than one key field position
	ds1.coGroup(ds2)
	.where(1, 3)
	.equalTo(
			new KeySelector<CustomType, Long>() {

					@Override
					public Long getKey(CustomType value) {
						return value.myLong;
					}
				}
			);
}
 

@Test(expected = InvalidProgramException.class)
public void testJoinKeyMixing3() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<CustomType> ds2 = env.fromCollection(customTypeData);

	// should not work, incompatible types
	ds1.join(ds2)
	.where(2)
	.equalTo(
			new KeySelector<CustomType, Long>() {

					@Override
					public Long getKey(CustomType value) {
						return value.myLong;
					}
				}
			);
}
 

/**
 * Defines that data is partitioned using a specific partitioning method
 * across input splits on the fields defined by field positions.
 * All records sharing the same key (combination) must be contained in a single input split.
 *
 * <p><b>
 *     IMPORTANT: Providing wrong information with SplitDataProperties can cause wrong results!
 * </b>
 *
 * @param partitionMethodId An ID for the method that was used to partition the data across splits.
 * @param partitionFields The field positions of the partitioning keys.
 * @return This SplitDataProperties object.
 */
public SplitDataProperties<T> splitsPartitionedBy(String partitionMethodId, int... partitionFields) {

	if (partitionFields == null) {
		throw new InvalidProgramException("PartitionFields may not be null.");
	} else if (partitionFields.length == 0) {
		throw new InvalidProgramException("PartitionFields may not be empty.");
	}

	this.splitPartitionKeys = getAllFlatKeys(partitionFields);
	if (partitionMethodId != null) {
		this.splitPartitioner = new SourcePartitionerMarker<>(partitionMethodId);
	} else {
		this.splitPartitioner = null;
	}

	return this;
}
 

@Test
public void testPOJOWithNestedArrayNoHashCodeKeyRejection() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<POJOWithHashCode> input = env.fromElements(
			new POJOWithHashCode(new int[] {1, 2}));

	TypeInformation<?> expectedTypeInfo = new TupleTypeInfo<Tuple1<int[]>>(
			PrimitiveArrayTypeInfo.INT_PRIMITIVE_ARRAY_TYPE_INFO);

	// adjust the rule
	expectedException.expect(InvalidProgramException.class);
	expectedException.expectMessage(new StringStartsWith("Type " + expectedTypeInfo + " cannot be used as key."));

	input.keyBy("id");
}
 

@Test(expected = InvalidProgramException.class)
public void testGroupAtomicTypeWithInvalid3() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<ArrayList<Integer>> dataSet = env.fromElements(new ArrayList<Integer>());

	dataSet.groupBy("*");
}
 

public void setDetachedPlan(FlinkPlan plan) {
	if (detachedPlan == null) {
		detachedPlan = plan;
	} else {
		throw new InvalidProgramException(DetachedJobExecutionResult.DETACHED_MESSAGE +
				DetachedJobExecutionResult.EXECUTE_TWICE_MESSAGE);
	}
}
 

private int[] getAllFlatKeys(String[] fieldExpressions) {

		int[] allKeys = null;

		for (String keyExp : fieldExpressions) {
			Keys.ExpressionKeys<T> ek = new Keys.ExpressionKeys<>(keyExp, this.type);
			int[] flatKeys = ek.computeLogicalKeyPositions();

			if (allKeys == null) {
				allKeys = flatKeys;
			} else {
				// check for duplicates
				for (int key1 : flatKeys) {
					for (int key2 : allKeys) {
						if (key1 == key2) {
							throw new InvalidProgramException("Duplicate fields in field expression " + keyExp);
						}
					}
				}
				// append flat keys
				int oldLength = allKeys.length;
				int newLength = oldLength + flatKeys.length;
				allKeys = Arrays.copyOf(allKeys, newLength);
				System.arraycopy(flatKeys, 0, allKeys, oldLength, flatKeys.length);
			}
		}

		return allKeys;
	}
 

@Test(expected = InvalidProgramException.class)
public void testDistinctOnNotKeyDataType() throws Exception {
   	/*
    	* should not work. NotComparable data type cannot be used as key
    	*/
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	NotComparable a = new NotComparable();
	List<NotComparable> l = new ArrayList<NotComparable>();
	l.add(a);

	DataSet<NotComparable> ds = env.fromCollection(l);
	DataSet<NotComparable> reduceDs = ds.distinct();

}
 

@Test(expected = InvalidProgramException.class)
public void testRangePartitionInIteration() throws Exception {

	// does not apply for collection execution
	if (super.mode == TestExecutionMode.COLLECTION) {
		throw new InvalidProgramException("Does not apply for collection execution");
	}

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSource<Long> source = env.generateSequence(0, 10000);

	DataSet<Tuple2<Long, String>> tuples = source.map(new MapFunction<Long, Tuple2<Long, String>>() {
		@Override
		public Tuple2<Long, String> map(Long v) throws Exception {
			return new Tuple2<>(v, Long.toString(v));
		}
	});

	DeltaIteration<Tuple2<Long, String>, Tuple2<Long, String>> it = tuples.iterateDelta(tuples, 10, 0);
	DataSet<Tuple2<Long, String>> body = it.getWorkset()
		.partitionByRange(1) // Verify that range partition is not allowed in iteration
		.join(it.getSolutionSet())
		.where(0).equalTo(0).projectFirst(0).projectSecond(1);
	DataSet<Tuple2<Long, String>> result = it.closeWith(body, body);

	result.collect(); // should fail
}
 

/**
 * Creates a new RemoteEnvironment that points to the master (JobManager) described by the
 * given host name and port.
 *
 * <p>Each program execution will have all the given JAR files in its classpath.
 *
 * @param host The host name or address of the master (JobManager), where the program should be executed.
 * @param port The port of the master (JobManager), where the program should be executed.
 * @param clientConfig The configuration used by the client that connects to the cluster.
 * @param jarFiles The JAR files with code that needs to be shipped to the cluster. If the program uses
 *                 user-defined functions, user-defined input formats, or any libraries, those must be
 *                 provided in the JAR files.
 * @param globalClasspaths The paths of directories and JAR files that are added to each user code
 *                 classloader on all nodes in the cluster. Note that the paths must specify a
 *                 protocol (e.g. file://) and be accessible on all nodes (e.g. by means of a NFS share).
 *                 The protocol must be supported by the {@link java.net.URLClassLoader}.
 */
public RemoteEnvironment(String host, int port, Configuration clientConfig,
		String[] jarFiles, URL[] globalClasspaths) {
	if (!ExecutionEnvironment.areExplicitEnvironmentsAllowed()) {
		throw new InvalidProgramException(
				"The RemoteEnvironment cannot be instantiated when running in a pre-defined context " +
						"(such as Command Line Client, Scala Shell, or TestEnvironment)");
	}
	if (host == null) {
		throw new NullPointerException("Host must not be null.");
	}
	if (port < 1 || port >= 0xffff) {
		throw new IllegalArgumentException("Port out of range");
	}

	this.host = host;
	this.port = port;
	this.clientConfiguration = clientConfig == null ? new Configuration() : clientConfig;
	if (jarFiles != null) {
		this.jarFiles = new ArrayList<>(jarFiles.length);
		for (String jarFile : jarFiles) {
			try {
				this.jarFiles.add(new File(jarFile).getAbsoluteFile().toURI().toURL());
			} catch (MalformedURLException e) {
				throw new IllegalArgumentException("JAR file path invalid", e);
			}
		}
	}
	else {
		this.jarFiles = Collections.emptyList();
	}

	if (globalClasspaths == null) {
		this.globalClasspaths = Collections.emptyList();
	} else {
		this.globalClasspaths = Arrays.asList(globalClasspaths);
	}
}
 

public Grouping(DataSet<T> set, Keys<T> keys) {
	if (set == null || keys == null) {
		throw new NullPointerException();
	}

	if (keys.isEmpty()) {
		throw new InvalidProgramException("The grouping keys must not be empty.");
	}

	this.inputDataSet = set;
	this.keys = keys;
}
 

private <T> TypeComparator<T> getTypeComparator(ExecutionConfig executionConfig, TypeInformation<T> inputType, int[] inputKeys, boolean[] inputSortDirections) {
	if (!(inputType instanceof CompositeType)) {
		throw new InvalidProgramException("Input types of coGroup must be composite types.");
	}

	return ((CompositeType<T>) inputType).createComparator(inputKeys, inputSortDirections, 0, executionConfig);
}
 

/**
 * Returns a new set containing the first n elements in this {@link DataSet}.
 *
 * @param n The desired number of elements.
 * @return A ReduceGroupOperator that represents the DataSet containing the elements.
 */
public GroupReduceOperator<T, T> first(int n) {
	if (n < 1) {
		throw new InvalidProgramException("Parameter n of first(n) must be at least 1.");
	}

	return reduceGroup(new FirstReducer<T>(n));
}
 

@Test(expected = InvalidProgramException.class)
public void testSortPartitionWithPositionKeys4() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs = env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);

	// must not work
	tupleDs.sortPartition(3, Order.ASCENDING);
}
 

private CoGroupOperatorSetsPredicate(Keys<I1> keys1) {
	if (keys1 == null) {
		throw new NullPointerException();
	}

	if (keys1.isEmpty()) {
		throw new InvalidProgramException("The co-group keys must not be empty.");
	}

	this.keys1 = keys1;
}
 

private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception {
	Operator<?> inputOp = sink.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The data sink " + sink.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> input = (List<IN>) execute(inputOp);
	
	@SuppressWarnings("unchecked")
	GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink;

	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
		
	if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}

	typedSink.executeOnCollections(input, ctx, executionConfig);
}
 

/**
 * Appends an additional sort order with the specified field in the specified order to the
 * local partition sorting of the DataSet.
 *
 * @param field The field index of the additional sort order of the local partition sorting.
 * @param order The order of the additional sort order of the local partition sorting.
 * @return The DataSet with sorted local partitions.
 */
public SortPartitionOperator<T> sortPartition(int field, Order order) {
	if (useKeySelector) {
		throw new InvalidProgramException("Expression keys cannot be appended after a KeySelector");
	}

	ensureSortableKey(field);
	keys.add(new Keys.ExpressionKeys<>(field, getType()));
	orders.add(order);

	return this;
}
 

private <T> TypeComparator<T> getTypeComparator(ExecutionConfig executionConfig, TypeInformation<T> inputType, int[] inputKeys, boolean[] inputSortDirections) {
	if (!(inputType instanceof CompositeType)) {
		throw new InvalidProgramException("Input types of coGroup must be composite types.");
	}

	return ((CompositeType<T>) inputType).createComparator(inputKeys, inputSortDirections, 0, executionConfig);
}
 

@Override
public <E> void validateCustomPartitioner(Partitioner<E> partitioner, TypeInformation<E> typeInfo) {

	if (keyFields.size() != 1) {
		throw new InvalidProgramException("Custom partitioners can only be used with keys that have one key field.");
	}

	if (typeInfo == null) {
		// try to extract key type from partitioner
		try {
			typeInfo = TypeExtractor.getPartitionerTypes(partitioner);
		}
		catch (Throwable t) {
			// best effort check, so we ignore exceptions
		}
	}

	if (typeInfo != null && !(typeInfo instanceof GenericTypeInfo)) {
		// only check type compatibility if type is known and not a generic type

		TypeInformation<?> keyType = keyFields.get(0).getType();
		if (!keyType.equals(typeInfo)) {
			throw new InvalidProgramException("The partitioner is incompatible with the key type. "
								+ "Partitioner type: " + typeInfo + " , key type: " + keyType);
		}
	}
}
 

@Test(expected = InvalidProgramException.class)
public void testJoinKeyInvalidAtomic4() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<Integer> ds2 = env.fromElements(0, 0, 0);

	ds1.join(ds2).where(0).equalTo("invalidKey");
}
 

@Test(expected = InvalidProgramException.class)
public void testJoinKeyMixedWrong() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
	DataSet<CustomType> ds2 = env.fromCollection(customTypeData);
	// wrongly mix String and Integer
	ds1.join(ds2).where("myString").equalTo(new KeySelector<CustomType, Integer>() {
		@Override
		public Integer getKey(CustomType value) throws Exception {
			return value.myInt;
		}
	});
}