org.apache.flink.api.java.typeutils.TupleTypeInfoBase Java Examples

The following examples show how to use org.apache.flink.api.java.typeutils.TupleTypeInfoBase. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FieldInfoUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
private static <A> List<FieldInfo> extractFieldInformation(
	TypeInformation<A> inputType,
	Expression[] exprs) {
	final List<FieldInfo> fieldInfos;
	if (inputType instanceof GenericTypeInfo && inputType.getTypeClass() == Row.class) {
		throw new ValidationException(
			"An input of GenericTypeInfo<Row> cannot be converted to Table. " +
				"Please specify the type of the input with a RowTypeInfo.");
	} else if (inputType instanceof TupleTypeInfoBase) {
		fieldInfos = extractFieldInfosFromTupleType((TupleTypeInfoBase<?>) inputType, exprs);
	} else if (inputType instanceof PojoTypeInfo) {
		fieldInfos = extractFieldInfosByNameReference((CompositeType<?>) inputType, exprs);
	} else {
		fieldInfos = extractFieldInfoFromAtomicType(inputType, exprs);
	}
	return fieldInfos;
}
 
Example #2
Source File: TupleCsvInputFormat.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private void configure(String lineDelimiter, String fieldDelimiter,
		TupleTypeInfoBase<OUT> tupleTypeInfo, boolean[] includedFieldsMask) {

	if (tupleTypeInfo.getArity() == 0) {
		throw new IllegalArgumentException("Tuple size must be greater than 0.");
	}

	if (includedFieldsMask == null) {
		includedFieldsMask = createDefaultMask(tupleTypeInfo.getArity());
	}

	tupleSerializer = (TupleSerializerBase<OUT>) tupleTypeInfo.createSerializer(new ExecutionConfig());

	setDelimiter(lineDelimiter);
	setFieldDelimiter(fieldDelimiter);

	Class<?>[] classes = new Class<?>[tupleTypeInfo.getArity()];

	for (int i = 0; i < tupleTypeInfo.getArity(); i++) {
		classes[i] = tupleTypeInfo.getTypeAt(i).getTypeClass();
	}

	setFieldsGeneric(includedFieldsMask, classes);
}
 
Example #3
Source File: AggregateOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Non grouped aggregation.
 */
public AggregateOperator(DataSet<IN> input, Aggregations function, int field, String aggregateLocationName) {
	super(Preconditions.checkNotNull(input), input.getType());
	Preconditions.checkNotNull(function);

	this.aggregateLocationName = aggregateLocationName;

	if (!input.getType().isTupleType()) {
		throw new InvalidProgramException("Aggregating on field positions is only possible on tuple data types.");
	}

	TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) input.getType();

	if (field < 0 || field >= inType.getArity()) {
		throw new IllegalArgumentException("Aggregation field position is out of range.");
	}

	AggregationFunctionFactory factory = function.getFactory();
	AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());

	// this is the first aggregation operator after a regular data set (non grouped aggregation)
	this.aggregationFunctions.add(aggFunct);
	this.fields.add(field);
	this.grouping = null;
}
 
Example #4
Source File: ScalaAggregateOperator.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Grouped aggregation.
 *
 * @param input
 * @param function
 * @param field
 */
public ScalaAggregateOperator(Grouping<IN> input, Aggregations function, int field) {
	super(Preconditions.checkNotNull(input).getInputDataSet(), input.getInputDataSet().getType());

	Preconditions.checkNotNull(function);

	if (!input.getInputDataSet().getType().isTupleType()) {
		throw new InvalidProgramException("Aggregating on field positions is only possible on tuple data types.");
	}

	TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) input.getInputDataSet().getType();

	if (field < 0 || field >= inType.getArity()) {
		throw new IllegalArgumentException("Aggregation field position is out of range.");
	}

	AggregationFunctionFactory factory = function.getFactory();
	AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());

	// set the aggregation fields
	this.aggregationFunctions.add(aggFunct);
	this.fields.add(field);
	this.grouping = input;
}
 
Example #5
Source File: AggregateOperator.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public AggregateOperator<IN> and(Aggregations function, int field) {
	Preconditions.checkNotNull(function);

	TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) getType();

	if (field < 0 || field >= inType.getArity()) {
		throw new IllegalArgumentException("Aggregation field position is out of range.");
	}

	AggregationFunctionFactory factory = function.getFactory();
	AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());

	this.aggregationFunctions.add(aggFunct);
	this.fields.add(field);

	return this;
}
 
Example #6
Source File: AggregateOperator.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Grouped aggregation.
 *
 * @param input
 * @param function
 * @param field
 */
public AggregateOperator(Grouping<IN> input, Aggregations function, int field, String aggregateLocationName) {
	super(Preconditions.checkNotNull(input).getInputDataSet(), input.getInputDataSet().getType());
	Preconditions.checkNotNull(function);

	this.aggregateLocationName = aggregateLocationName;

	if (!input.getInputDataSet().getType().isTupleType()) {
		throw new InvalidProgramException("Aggregating on field positions is only possible on tuple data types.");
	}

	TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) input.getInputDataSet().getType();

	if (field < 0 || field >= inType.getArity()) {
		throw new IllegalArgumentException("Aggregation field position is out of range.");
	}

	AggregationFunctionFactory factory = function.getFactory();
	AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());

	// set the aggregation fields
	this.aggregationFunctions.add(aggFunct);
	this.fields.add(field);
	this.grouping = input;
}
 
Example #7
Source File: AggregateOperator.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Non grouped aggregation.
 */
public AggregateOperator(DataSet<IN> input, Aggregations function, int field, String aggregateLocationName) {
	super(Preconditions.checkNotNull(input), input.getType());
	Preconditions.checkNotNull(function);

	this.aggregateLocationName = aggregateLocationName;

	if (!input.getType().isTupleType()) {
		throw new InvalidProgramException("Aggregating on field positions is only possible on tuple data types.");
	}

	TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) input.getType();

	if (field < 0 || field >= inType.getArity()) {
		throw new IllegalArgumentException("Aggregation field position is out of range.");
	}

	AggregationFunctionFactory factory = function.getFactory();
	AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());

	// this is the first aggregation operator after a regular data set (non grouped aggregation)
	this.aggregationFunctions.add(aggFunct);
	this.fields.add(field);
	this.grouping = null;
}
 
Example #8
Source File: FieldAccessor.java    From flink with Apache License 2.0 6 votes vote down vote up
RecursiveProductFieldAccessor(int pos, TypeInformation<T> typeInfo, FieldAccessor<R, F> innerAccessor, ExecutionConfig config) {
	int arity = ((TupleTypeInfoBase) typeInfo).getArity();
	if (pos < 0 || pos >= arity) {
		throw new CompositeType.InvalidFieldReferenceException(
			"Tried to select " + ((Integer) pos).toString() + ". field on \"" +
				typeInfo.toString() + "\", which is an invalid index.");
	}
	checkNotNull(typeInfo, "typeInfo must not be null.");
	checkNotNull(innerAccessor, "innerAccessor must not be null.");

	this.pos = pos;
	this.serializer = (TupleSerializerBase<T>) typeInfo.createSerializer(config);
	this.length = this.serializer.getArity();
	this.fields = new Object[this.length];
	this.innerAccessor = innerAccessor;
	this.fieldType = innerAccessor.getFieldType();
}
 
Example #9
Source File: DataSetUtils.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Summarize a DataSet of Tuples by collecting single pass statistics for all columns.
 *
 * <p>Example usage:
 * <pre>
 * {@code
 * Dataset<Tuple3<Double, String, Boolean>> input = // [...]
 * Tuple3<NumericColumnSummary,StringColumnSummary, BooleanColumnSummary> summary = DataSetUtils.summarize(input)
 *
 * summary.f0.getStandardDeviation()
 * summary.f1.getMaxLength()
 * }
 * </pre>
 * @return the summary as a Tuple the same width as input rows
 */
public static <R extends Tuple, T extends Tuple> R summarize(DataSet<T> input) throws Exception {
	if (!input.getType().isTupleType()) {
		throw new IllegalArgumentException("summarize() is only implemented for DataSet's of Tuples");
	}
	final TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) input.getType();
	DataSet<TupleSummaryAggregator<R>> result = input.mapPartition(new MapPartitionFunction<T, TupleSummaryAggregator<R>>() {
		@Override
		public void mapPartition(Iterable<T> values, Collector<TupleSummaryAggregator<R>> out) throws Exception {
			TupleSummaryAggregator<R> aggregator = SummaryAggregatorFactory.create(inType);
			for (Tuple value : values) {
				aggregator.aggregate(value);
			}
			out.collect(aggregator);
		}
	}).reduce(new ReduceFunction<TupleSummaryAggregator<R>>() {
		@Override
		public TupleSummaryAggregator<R> reduce(TupleSummaryAggregator<R> agg1, TupleSummaryAggregator<R> agg2) throws Exception {
			agg1.combine(agg2);
			return agg1;
		}
	});
	return result.collect().get(0).result();
}
 
Example #10
Source File: ScalaAggregateOperator.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public ScalaAggregateOperator<IN> and(Aggregations function, int field) {
	Preconditions.checkNotNull(function);

	TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) getType();

	if (field < 0 || field >= inType.getArity()) {
		throw new IllegalArgumentException("Aggregation field position is out of range.");
	}

	AggregationFunctionFactory factory = function.getFactory();
	AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());

	this.aggregationFunctions.add(aggFunct);
	this.fields.add(field);

	return this;
}
 
Example #11
Source File: ScalaAggregateOperator.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Non grouped aggregation.
 */
public ScalaAggregateOperator(org.apache.flink.api.java.DataSet<IN> input, Aggregations function, int field) {
	super(Preconditions.checkNotNull(input), input.getType());

	Preconditions.checkNotNull(function);

	if (!input.getType().isTupleType()) {
		throw new InvalidProgramException("Aggregating on field positions is only possible on tuple data types.");
	}

	TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) input.getType();

	if (field < 0 || field >= inType.getArity()) {
		throw new IllegalArgumentException("Aggregation field position is out of range.");
	}

	AggregationFunctionFactory factory = function.getFactory();
	AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());

	// this is the first aggregation operator after a regular data set (non grouped aggregation)
	this.aggregationFunctions.add(aggFunct);
	this.fields.add(field);
	this.grouping = null;
}
 
Example #12
Source File: SequenceFileWriter.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void setInputType(TypeInformation<?> type, ExecutionConfig executionConfig) {
	if (!type.isTupleType()) {
		throw new IllegalArgumentException("Input TypeInformation is not a tuple type.");
	}

	TupleTypeInfoBase<?> tupleType = (TupleTypeInfoBase<?>) type;

	if (tupleType.getArity() != 2) {
		throw new IllegalArgumentException("Input TypeInformation must be a Tuple2 type.");
	}

	TypeInformation<K> keyType = tupleType.getTypeAt(0);
	TypeInformation<V> valueType = tupleType.getTypeAt(1);

	this.keyClass = keyType.getTypeClass();
	this.valueClass = valueType.getTypeClass();
}
 
Example #13
Source File: DataSetUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Summarize a DataSet of Tuples by collecting single pass statistics for all columns.
 *
 * <p>Example usage:
 * <pre>
 * {@code
 * Dataset<Tuple3<Double, String, Boolean>> input = // [...]
 * Tuple3<NumericColumnSummary,StringColumnSummary, BooleanColumnSummary> summary = DataSetUtils.summarize(input)
 *
 * summary.f0.getStandardDeviation()
 * summary.f1.getMaxLength()
 * }
 * </pre>
 * @return the summary as a Tuple the same width as input rows
 */
public static <R extends Tuple, T extends Tuple> R summarize(DataSet<T> input) throws Exception {
	if (!input.getType().isTupleType()) {
		throw new IllegalArgumentException("summarize() is only implemented for DataSet's of Tuples");
	}
	final TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) input.getType();
	DataSet<TupleSummaryAggregator<R>> result = input.mapPartition(new MapPartitionFunction<T, TupleSummaryAggregator<R>>() {
		@Override
		public void mapPartition(Iterable<T> values, Collector<TupleSummaryAggregator<R>> out) throws Exception {
			TupleSummaryAggregator<R> aggregator = SummaryAggregatorFactory.create(inType);
			for (Tuple value : values) {
				aggregator.aggregate(value);
			}
			out.collect(aggregator);
		}
	}).reduce(new ReduceFunction<TupleSummaryAggregator<R>>() {
		@Override
		public TupleSummaryAggregator<R> reduce(TupleSummaryAggregator<R> agg1, TupleSummaryAggregator<R> agg2) throws Exception {
			agg1.combine(agg2);
			return agg1;
		}
	});
	return result.collect().get(0).result();
}
 
Example #14
Source File: ScalaAggregateOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
public ScalaAggregateOperator<IN> and(Aggregations function, int field) {
	Preconditions.checkNotNull(function);

	TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) getType();

	if (field < 0 || field >= inType.getArity()) {
		throw new IllegalArgumentException("Aggregation field position is out of range.");
	}

	AggregationFunctionFactory factory = function.getFactory();
	AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());

	this.aggregationFunctions.add(aggFunct);
	this.fields.add(field);

	return this;
}
 
Example #15
Source File: FieldInfoUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Reference input fields by name:
 * All fields in the schema definition are referenced by name
 * (and possibly renamed using an alias (as). In this mode, fields can be reordered and
 * projected out. Moreover, we can define proctime and rowtime attributes at arbitrary
 * positions using arbitrary names (except those that exist in the result schema). This mode
 * can be used for any input type, including POJOs.
 *
 * <p>Reference input fields by position:
 * In this mode, fields are simply renamed. Event-time attributes can
 * replace the field on their position in the input data (if it is of correct type) or be
 * appended at the end. Proctime attributes must be appended at the end. This mode can only be
 * used if the input type has a defined field order (tuple, case class, Row) and no of fields
 * references a field of the input type.
 */
public static boolean isReferenceByPosition(CompositeType<?> ct, Expression[] fields) {
	if (!(ct instanceof TupleTypeInfoBase)) {
		return false;
	}

	List<String> inputNames = Arrays.asList(ct.getFieldNames());

	// Use the by-position mode if no of the fields exists in the input.
	// This prevents confusing cases like ('f2, 'f0, 'myName) for a Tuple3 where fields are renamed
	// by position but the user might assume reordering instead of renaming.
	return Arrays.stream(fields).allMatch(f -> {
		if (f instanceof UnresolvedReferenceExpression) {
			return !inputNames.contains(((UnresolvedReferenceExpression) f).getName());
		}

		return true;
	});
}
 
Example #16
Source File: SequenceFileWriter.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public void setInputType(TypeInformation<?> type, ExecutionConfig executionConfig) {
	if (!type.isTupleType()) {
		throw new IllegalArgumentException("Input TypeInformation is not a tuple type.");
	}

	TupleTypeInfoBase<?> tupleType = (TupleTypeInfoBase<?>) type;

	if (tupleType.getArity() != 2) {
		throw new IllegalArgumentException("Input TypeInformation must be a Tuple2 type.");
	}

	TypeInformation<K> keyType = tupleType.getTypeAt(0);
	TypeInformation<V> valueType = tupleType.getTypeAt(1);

	this.keyClass = keyType.getTypeClass();
	this.valueClass = valueType.getTypeClass();
}
 
Example #17
Source File: ScalaAggregateOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Grouped aggregation.
 *
 * @param input
 * @param function
 * @param field
 */
public ScalaAggregateOperator(Grouping<IN> input, Aggregations function, int field) {
	super(Preconditions.checkNotNull(input).getInputDataSet(), input.getInputDataSet().getType());

	Preconditions.checkNotNull(function);

	if (!input.getInputDataSet().getType().isTupleType()) {
		throw new InvalidProgramException("Aggregating on field positions is only possible on tuple data types.");
	}

	TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) input.getInputDataSet().getType();

	if (field < 0 || field >= inType.getArity()) {
		throw new IllegalArgumentException("Aggregation field position is out of range.");
	}

	AggregationFunctionFactory factory = function.getFactory();
	AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());

	// set the aggregation fields
	this.aggregationFunctions.add(aggFunct);
	this.fields.add(field);
	this.grouping = input;
}
 
Example #18
Source File: ScalaAggregateOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Non grouped aggregation.
 */
public ScalaAggregateOperator(org.apache.flink.api.java.DataSet<IN> input, Aggregations function, int field) {
	super(Preconditions.checkNotNull(input), input.getType());

	Preconditions.checkNotNull(function);

	if (!input.getType().isTupleType()) {
		throw new InvalidProgramException("Aggregating on field positions is only possible on tuple data types.");
	}

	TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) input.getType();

	if (field < 0 || field >= inType.getArity()) {
		throw new IllegalArgumentException("Aggregation field position is out of range.");
	}

	AggregationFunctionFactory factory = function.getFactory();
	AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());

	// this is the first aggregation operator after a regular data set (non grouped aggregation)
	this.aggregationFunctions.add(aggFunct);
	this.fields.add(field);
	this.grouping = null;
}
 
Example #19
Source File: ScalaAggregateOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Grouped aggregation.
 *
 * @param input
 * @param function
 * @param field
 */
public ScalaAggregateOperator(Grouping<IN> input, Aggregations function, int field) {
	super(Preconditions.checkNotNull(input).getInputDataSet(), input.getInputDataSet().getType());

	Preconditions.checkNotNull(function);

	if (!input.getInputDataSet().getType().isTupleType()) {
		throw new InvalidProgramException("Aggregating on field positions is only possible on tuple data types.");
	}

	TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) input.getInputDataSet().getType();

	if (field < 0 || field >= inType.getArity()) {
		throw new IllegalArgumentException("Aggregation field position is out of range.");
	}

	AggregationFunctionFactory factory = function.getFactory();
	AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());

	// set the aggregation fields
	this.aggregationFunctions.add(aggFunct);
	this.fields.add(field);
	this.grouping = input;
}
 
Example #20
Source File: ScalaAggregateOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
public ScalaAggregateOperator<IN> and(Aggregations function, int field) {
	Preconditions.checkNotNull(function);

	TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) getType();

	if (field < 0 || field >= inType.getArity()) {
		throw new IllegalArgumentException("Aggregation field position is out of range.");
	}

	AggregationFunctionFactory factory = function.getFactory();
	AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());

	this.aggregationFunctions.add(aggFunct);
	this.fields.add(field);

	return this;
}
 
Example #21
Source File: DataSetUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Summarize a DataSet of Tuples by collecting single pass statistics for all columns.
 *
 * <p>Example usage:
 * <pre>
 * {@code
 * Dataset<Tuple3<Double, String, Boolean>> input = // [...]
 * Tuple3<NumericColumnSummary,StringColumnSummary, BooleanColumnSummary> summary = DataSetUtils.summarize(input)
 *
 * summary.f0.getStandardDeviation()
 * summary.f1.getMaxLength()
 * }
 * </pre>
 * @return the summary as a Tuple the same width as input rows
 */
public static <R extends Tuple, T extends Tuple> R summarize(DataSet<T> input) throws Exception {
	if (!input.getType().isTupleType()) {
		throw new IllegalArgumentException("summarize() is only implemented for DataSet's of Tuples");
	}
	final TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) input.getType();
	DataSet<TupleSummaryAggregator<R>> result = input.mapPartition(new MapPartitionFunction<T, TupleSummaryAggregator<R>>() {
		@Override
		public void mapPartition(Iterable<T> values, Collector<TupleSummaryAggregator<R>> out) throws Exception {
			TupleSummaryAggregator<R> aggregator = SummaryAggregatorFactory.create(inType);
			for (Tuple value : values) {
				aggregator.aggregate(value);
			}
			out.collect(aggregator);
		}
	}).reduce(new ReduceFunction<TupleSummaryAggregator<R>>() {
		@Override
		public TupleSummaryAggregator<R> reduce(TupleSummaryAggregator<R> agg1, TupleSummaryAggregator<R> agg2) throws Exception {
			agg1.combine(agg2);
			return agg1;
		}
	});
	return result.collect().get(0).result();
}
 
Example #22
Source File: ScalaAggregateOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Non grouped aggregation.
 */
public ScalaAggregateOperator(org.apache.flink.api.java.DataSet<IN> input, Aggregations function, int field) {
	super(Preconditions.checkNotNull(input), input.getType());

	Preconditions.checkNotNull(function);

	if (!input.getType().isTupleType()) {
		throw new InvalidProgramException("Aggregating on field positions is only possible on tuple data types.");
	}

	TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) input.getType();

	if (field < 0 || field >= inType.getArity()) {
		throw new IllegalArgumentException("Aggregation field position is out of range.");
	}

	AggregationFunctionFactory factory = function.getFactory();
	AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());

	// this is the first aggregation operator after a regular data set (non grouped aggregation)
	this.aggregationFunctions.add(aggFunct);
	this.fields.add(field);
	this.grouping = null;
}
 
Example #23
Source File: AggregateOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Non grouped aggregation.
 */
public AggregateOperator(DataSet<IN> input, Aggregations function, int field, String aggregateLocationName) {
	super(Preconditions.checkNotNull(input), input.getType());
	Preconditions.checkNotNull(function);

	this.aggregateLocationName = aggregateLocationName;

	if (!input.getType().isTupleType()) {
		throw new InvalidProgramException("Aggregating on field positions is only possible on tuple data types.");
	}

	TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) input.getType();

	if (field < 0 || field >= inType.getArity()) {
		throw new IllegalArgumentException("Aggregation field position is out of range.");
	}

	AggregationFunctionFactory factory = function.getFactory();
	AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());

	// this is the first aggregation operator after a regular data set (non grouped aggregation)
	this.aggregationFunctions.add(aggFunct);
	this.fields.add(field);
	this.grouping = null;
}
 
Example #24
Source File: AggregateOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Grouped aggregation.
 *
 * @param input
 * @param function
 * @param field
 */
public AggregateOperator(Grouping<IN> input, Aggregations function, int field, String aggregateLocationName) {
	super(Preconditions.checkNotNull(input).getInputDataSet(), input.getInputDataSet().getType());
	Preconditions.checkNotNull(function);

	this.aggregateLocationName = aggregateLocationName;

	if (!input.getInputDataSet().getType().isTupleType()) {
		throw new InvalidProgramException("Aggregating on field positions is only possible on tuple data types.");
	}

	TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) input.getInputDataSet().getType();

	if (field < 0 || field >= inType.getArity()) {
		throw new IllegalArgumentException("Aggregation field position is out of range.");
	}

	AggregationFunctionFactory factory = function.getFactory();
	AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());

	// set the aggregation fields
	this.aggregationFunctions.add(aggFunct);
	this.fields.add(field);
	this.grouping = input;
}
 
Example #25
Source File: AggregateOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
public AggregateOperator<IN> and(Aggregations function, int field) {
	Preconditions.checkNotNull(function);

	TupleTypeInfoBase<?> inType = (TupleTypeInfoBase<?>) getType();

	if (field < 0 || field >= inType.getArity()) {
		throw new IllegalArgumentException("Aggregation field position is out of range.");
	}

	AggregationFunctionFactory factory = function.getFactory();
	AggregationFunction<?> aggFunct = factory.createAggregationFunction(inType.getTypeAt(field).getTypeClass());

	this.aggregationFunctions.add(aggFunct);
	this.fields.add(field);

	return this;
}
 
Example #26
Source File: FieldInfoUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
private static <A> List<FieldInfo> extractFieldInformation(
		TypeInformation<A> inputType,
		Expression[] exprs) {
	final List<FieldInfo> fieldInfos;
	if (inputType instanceof GenericTypeInfo && inputType.getTypeClass() == Row.class) {
		throw new ValidationException(
			"An input of GenericTypeInfo<Row> cannot be converted to Table. " +
				"Please specify the type of the input with a RowTypeInfo.");
	} else if (inputType instanceof TupleTypeInfoBase) {
		fieldInfos = extractFieldInfosFromTupleType((TupleTypeInfoBase<?>) inputType, exprs);
	} else if (inputType instanceof PojoTypeInfo) {
		fieldInfos = extractFieldInfosByNameReference((CompositeType<?>) inputType, exprs);
	} else {
		fieldInfos = extractFieldInfoFromAtomicType(inputType, exprs);
	}
	return fieldInfos;
}
 
Example #27
Source File: FieldInfoUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Reference input fields by name:
 * All fields in the schema definition are referenced by name
 * (and possibly renamed using an alias (as). In this mode, fields can be reordered and
 * projected out. Moreover, we can define proctime and rowtime attributes at arbitrary
 * positions using arbitrary names (except those that exist in the result schema). This mode
 * can be used for any input type, including POJOs.
 *
 * <p>Reference input fields by position:
 * In this mode, fields are simply renamed. Event-time attributes can
 * replace the field on their position in the input data (if it is of correct type) or be
 * appended at the end. Proctime attributes must be appended at the end. This mode can only be
 * used if the input type has a defined field order (tuple, case class, Row) and no of fields
 * references a field of the input type.
 */
private static boolean isReferenceByPosition(CompositeType<?> ct, Expression[] fields) {
	if (!(ct instanceof TupleTypeInfoBase)) {
		return false;
	}

	List<String> inputNames = Arrays.asList(ct.getFieldNames());

	// Use the by-position mode if no of the fields exists in the input.
	// This prevents confusing cases like ('f2, 'f0, 'myName) for a Tuple3 where fields are renamed
	// by position but the user might assume reordering instead of renaming.
	return Arrays.stream(fields).allMatch(f -> {
		if (f instanceof UnresolvedCallExpression &&
			((UnresolvedCallExpression) f).getFunctionDefinition() == BuiltInFunctionDefinitions.AS &&
			f.getChildren().get(0) instanceof UnresolvedReferenceExpression) {
			return false;
		}
		if (f instanceof UnresolvedReferenceExpression) {
			return !inputNames.contains(((UnresolvedReferenceExpression) f).getName());
		}

		return true;
	});
}
 
Example #28
Source File: SequenceFileWriter.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void setInputType(TypeInformation<?> type, ExecutionConfig executionConfig) {
	if (!type.isTupleType()) {
		throw new IllegalArgumentException("Input TypeInformation is not a tuple type.");
	}

	TupleTypeInfoBase<?> tupleType = (TupleTypeInfoBase<?>) type;

	if (tupleType.getArity() != 2) {
		throw new IllegalArgumentException("Input TypeInformation must be a Tuple2 type.");
	}

	TypeInformation<K> keyType = tupleType.getTypeAt(0);
	TypeInformation<V> valueType = tupleType.getTypeAt(1);

	this.keyClass = keyType.getTypeClass();
	this.valueClass = valueType.getTypeClass();
}
 
Example #29
Source File: FieldAccessor.java    From flink with Apache License 2.0 6 votes vote down vote up
RecursiveProductFieldAccessor(int pos, TypeInformation<T> typeInfo, FieldAccessor<R, F> innerAccessor, ExecutionConfig config) {
	int arity = ((TupleTypeInfoBase) typeInfo).getArity();
	if (pos < 0 || pos >= arity) {
		throw new CompositeType.InvalidFieldReferenceException(
			"Tried to select " + ((Integer) pos).toString() + ". field on \"" +
				typeInfo.toString() + "\", which is an invalid index.");
	}
	checkNotNull(typeInfo, "typeInfo must not be null.");
	checkNotNull(innerAccessor, "innerAccessor must not be null.");

	this.pos = pos;
	this.fieldType = ((TupleTypeInfoBase<T>) typeInfo).getTypeAt(pos);
	this.serializer = (TupleSerializerBase<T>) typeInfo.createSerializer(config);
	this.length = this.serializer.getArity();
	this.fields = new Object[this.length];
	this.innerAccessor = innerAccessor;
}
 
Example #30
Source File: TupleCsvInputFormat.java    From flink with Apache License 2.0 6 votes vote down vote up
private void configure(String lineDelimiter, String fieldDelimiter,
		TupleTypeInfoBase<OUT> tupleTypeInfo, boolean[] includedFieldsMask) {

	if (tupleTypeInfo.getArity() == 0) {
		throw new IllegalArgumentException("Tuple size must be greater than 0.");
	}

	if (includedFieldsMask == null) {
		includedFieldsMask = createDefaultMask(tupleTypeInfo.getArity());
	}

	tupleSerializer = (TupleSerializerBase<OUT>) tupleTypeInfo.createSerializer(new ExecutionConfig());

	setDelimiter(lineDelimiter);
	setFieldDelimiter(fieldDelimiter);

	Class<?>[] classes = new Class<?>[tupleTypeInfo.getArity()];

	for (int i = 0; i < tupleTypeInfo.getArity(); i++) {
		classes[i] = tupleTypeInfo.getTypeAt(i).getTypeClass();
	}

	setFieldsGeneric(includedFieldsMask, classes);
}