org.apache.flink.api.java.typeutils.ListTypeInfo Java Examples

The following examples show how to use org.apache.flink.api.java.typeutils.ListTypeInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ProcTimeRangeBoundedPrecedingFunction.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
	function.open(new PerKeyStateDataViewStore(getRuntimeContext()));

	output = new JoinedRowData();

	// input element are all binary row as they are came from network
	RowDataTypeInfo inputType = new RowDataTypeInfo(inputFieldTypes);
	// we keep the elements received in a map state indexed based on their ingestion time
	ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<>(inputType);
	MapStateDescriptor<Long, List<RowData>> mapStateDescriptor = new MapStateDescriptor<>(
		"inputState", BasicTypeInfo.LONG_TYPE_INFO, rowListTypeInfo);
	inputState = getRuntimeContext().getMapState(mapStateDescriptor);

	RowDataTypeInfo accTypeInfo = new RowDataTypeInfo(accTypes);
	ValueStateDescriptor<RowData> stateDescriptor =
		new ValueStateDescriptor<RowData>("accState", accTypeInfo);
	accState = getRuntimeContext().getState(stateDescriptor);

	ValueStateDescriptor<Long> cleanupTsStateDescriptor = new ValueStateDescriptor<>(
		"cleanupTsState",
		Types.LONG
	);
	this.cleanupTsState = getRuntimeContext().getState(cleanupTsStateDescriptor);
}
 
Example #2
Source File: RowTimeSortOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void open() throws Exception {
	super.open();

	LOG.info("Opening RowTimeSortOperator");
	if (gComparator != null) {
		comparator = gComparator.newInstance(getContainingTask().getUserCodeClassLoader());
		gComparator = null;
	}

	BasicTypeInfo<Long> keyTypeInfo = BasicTypeInfo.LONG_TYPE_INFO;
	ListTypeInfo<RowData> valueTypeInfo = new ListTypeInfo<>(inputRowType);
	MapStateDescriptor<Long, List<RowData>> mapStateDescriptor = new MapStateDescriptor<>(
			"dataState", keyTypeInfo, valueTypeInfo);
	dataState = getRuntimeContext().getMapState(mapStateDescriptor);

	ValueStateDescriptor<Long> lastTriggeringTsDescriptor = new ValueStateDescriptor<>("lastTriggeringTsState",
			Long.class);
	lastTriggeringTsState = getRuntimeContext().getState(lastTriggeringTsDescriptor);
}
 
Example #3
Source File: LastValueWithRetractAggFunction.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public TypeInformation<GenericRowData> getAccumulatorType() {
	LogicalType[] fieldTypes = new LogicalType[] {
			fromTypeInfoToLogicalType(getResultType()),
			new BigIntType(),
			new TypeInformationRawType<>(new MapViewTypeInfo<>(getResultType(), new ListTypeInfo<>(Types.LONG), false, false)),
			new TypeInformationRawType<>(new MapViewTypeInfo<>(Types.LONG, new ListTypeInfo<>(getResultType()), false, false))
	};

	String[] fieldNames = new String[] {
			"lastValue",
			"lastOrder",
			"valueToOrderMapView",
			"orderToValueMapView"
	};

	return (TypeInformation) new RowDataTypeInfo(fieldTypes, fieldNames);
}
 
Example #4
Source File: LastValueWithRetractAggFunction.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public TypeInformation<GenericRow> getAccumulatorType() {
	LogicalType[] fieldTypes = new LogicalType[] {
			fromTypeInfoToLogicalType(getResultType()),
			new BigIntType(),
			new TypeInformationAnyType<>(new MapViewTypeInfo<>(getResultType(), new ListTypeInfo<>(Types.LONG), false, false)),
			new TypeInformationAnyType<>(new MapViewTypeInfo<>(Types.LONG, new ListTypeInfo<>(getResultType()), false, false))
	};

	String[] fieldNames = new String[] {
			"lastValue",
			"lastOrder",
			"valueToOrderMapView",
			"orderToValueMapView"
	};

	return (TypeInformation) new BaseRowTypeInfo(fieldTypes, fieldNames);
}
 
Example #5
Source File: LastValueWithRetractAggFunction.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public GenericRow createAccumulator() {
	// The accumulator schema:
	// lastValue: T
	// lastOrder: Long
	// valueToOrderMap: BinaryGeneric<MapView<T, List<Long>>>
	// orderToValueMap: BinaryGeneric<MapView<Long, List<T>>>
	GenericRow acc = new GenericRow(4);
	acc.setField(0, null);
	acc.setField(1, null);
	acc.setField(2, new BinaryGeneric<>(
			new MapView<>(getResultType(), new ListTypeInfo<>(Types.LONG)), getValueToOrderMapViewSerializer()));
	acc.setField(3, new BinaryGeneric<>(
			new MapView<>(Types.LONG, new ListTypeInfo<>(getResultType())), getOrderToValueMapViewSerializer()));
	return acc;
}
 
Example #6
Source File: FirstValueWithRetractAggFunction.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public TypeInformation<GenericRow> getAccumulatorType() {
	LogicalType[] fieldTypes = new LogicalType[] {
			fromTypeInfoToLogicalType(getResultType()),
			new BigIntType(),
			new TypeInformationAnyType<>(new MapViewTypeInfo<>(getResultType(), new ListTypeInfo<>(Types.LONG), false, false)),
			new TypeInformationAnyType<>(new MapViewTypeInfo<>(Types.LONG, new ListTypeInfo<>(getResultType()), false, false))
	};

	String[] fieldNames = new String[] {
			"firstValue",
			"firstOrder",
			"valueToOrderMapView",
			"orderToValueMapView"
	};

	return (TypeInformation) new BaseRowTypeInfo(fieldTypes, fieldNames);
}
 
Example #7
Source File: FirstValueWithRetractAggFunction.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public GenericRow createAccumulator() {
	// The accumulator schema:
	// firstValue: T
	// fistOrder: Long
	// valueToOrderMap: BinaryGeneric<MapView<T, List<Long>>>
	// orderToValueMap: BinaryGeneric<MapView<Long, List<T>>>
	GenericRow acc = new GenericRow(4);
	acc.setField(0, null);
	acc.setField(1, null);
	acc.setField(2, new BinaryGeneric<>(
		new MapView<>(getResultType(), new ListTypeInfo<>(Types.LONG)),
		getValueToOrderMapViewSerializer()));
	acc.setField(3, new BinaryGeneric<>(
		new MapView<>(Types.LONG, new ListTypeInfo<>(getResultType())),
		getOrderToValueMapViewSerializer()));
	return acc;
}
 
Example #8
Source File: RetractableTopNFunction.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	// compile equaliser
	equaliser = generatedEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
	generatedEqualiser = null;

	ListTypeInfo<BaseRow> valueTypeInfo = new ListTypeInfo<>(inputRowType);
	MapStateDescriptor<BaseRow, List<BaseRow>> mapStateDescriptor = new MapStateDescriptor<>(
			"data-state", sortKeyType, valueTypeInfo);
	dataState = getRuntimeContext().getMapState(mapStateDescriptor);

	ValueStateDescriptor<SortedMap<BaseRow, Long>> valueStateDescriptor = new ValueStateDescriptor<>(
			"sorted-map",
			new SortedMapTypeInfo<>(sortKeyType, BasicTypeInfo.LONG_TYPE_INFO, serializableComparator));
	treeMap = getRuntimeContext().getState(valueStateDescriptor);
}
 
Example #9
Source File: AbstractRowTimeUnboundedPrecedingOver.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
	function.open(new PerKeyStateDataViewStore(getRuntimeContext()));

	output = new JoinedRow();

	sortedTimestamps = new LinkedList<Long>();

	// initialize accumulator state
	BaseRowTypeInfo accTypeInfo = new BaseRowTypeInfo(accTypes);
	ValueStateDescriptor<BaseRow> accStateDesc =
		new ValueStateDescriptor<BaseRow>("accState", accTypeInfo);
	accState = getRuntimeContext().getState(accStateDesc);

	// input element are all binary row as they are came from network
	BaseRowTypeInfo inputType = new BaseRowTypeInfo(inputFieldTypes);
	ListTypeInfo<BaseRow> rowListTypeInfo = new ListTypeInfo<BaseRow>(inputType);
	MapStateDescriptor<Long, List<BaseRow>> inputStateDesc = new MapStateDescriptor<Long, List<BaseRow>>(
		"inputState",
		Types.LONG,
		rowListTypeInfo);
	inputState = getRuntimeContext().getMapState(inputStateDesc);

	initCleanupTimeState("RowTimeUnboundedOverCleanupTime");
}
 
Example #10
Source File: ProcTimeRangeBoundedPrecedingFunction.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
	function.open(new PerKeyStateDataViewStore(getRuntimeContext()));

	output = new JoinedRow();

	// input element are all binary row as they are came from network
	BaseRowTypeInfo inputType = new BaseRowTypeInfo(inputFieldTypes);
	// we keep the elements received in a map state indexed based on their ingestion time
	ListTypeInfo<BaseRow> rowListTypeInfo = new ListTypeInfo<BaseRow>(inputType);
	MapStateDescriptor<Long, List<BaseRow>> mapStateDescriptor = new MapStateDescriptor<Long, List<BaseRow>>(
		"inputState", BasicTypeInfo.LONG_TYPE_INFO, rowListTypeInfo);
	inputState = getRuntimeContext().getMapState(mapStateDescriptor);

	BaseRowTypeInfo accTypeInfo = new BaseRowTypeInfo(accTypes);
	ValueStateDescriptor<BaseRow> stateDescriptor =
		new ValueStateDescriptor<BaseRow>("accState", accTypeInfo);
	accState = getRuntimeContext().getState(stateDescriptor);

	initCleanupTimeState("ProcTimeBoundedRangeOverCleanupTime");
}
 
Example #11
Source File: AbstractRowTimeUnboundedPrecedingOver.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
	function.open(new PerKeyStateDataViewStore(getRuntimeContext()));

	output = new JoinedRowData();

	sortedTimestamps = new LinkedList<Long>();

	// initialize accumulator state
	RowDataTypeInfo accTypeInfo = new RowDataTypeInfo(accTypes);
	ValueStateDescriptor<RowData> accStateDesc =
		new ValueStateDescriptor<RowData>("accState", accTypeInfo);
	accState = getRuntimeContext().getState(accStateDesc);

	// input element are all binary row as they are came from network
	RowDataTypeInfo inputType = new RowDataTypeInfo(inputFieldTypes);
	ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
	MapStateDescriptor<Long, List<RowData>> inputStateDesc = new MapStateDescriptor<Long, List<RowData>>(
		"inputState",
		Types.LONG,
		rowListTypeInfo);
	inputState = getRuntimeContext().getMapState(inputStateDesc);

	initCleanupTimeState("RowTimeUnboundedOverCleanupTime");
}
 
Example #12
Source File: RetractableTopNFunction.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	// compile equaliser
	equaliser = generatedEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
	generatedEqualiser = null;

	ListTypeInfo<RowData> valueTypeInfo = new ListTypeInfo<>(inputRowType);
	MapStateDescriptor<RowData, List<RowData>> mapStateDescriptor = new MapStateDescriptor<>(
			"data-state", sortKeyType, valueTypeInfo);
	dataState = getRuntimeContext().getMapState(mapStateDescriptor);

	ValueStateDescriptor<SortedMap<RowData, Long>> valueStateDescriptor = new ValueStateDescriptor<>(
			"sorted-map",
			new SortedMapTypeInfo<>(sortKeyType, BasicTypeInfo.LONG_TYPE_INFO, serializableComparator));
	treeMap = getRuntimeContext().getState(valueStateDescriptor);
}
 
Example #13
Source File: RowTimeSortOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void open() throws Exception {
	super.open();

	LOG.info("Opening RowTimeSortOperator");
	if (gComparator != null) {
		comparator = gComparator.newInstance(getContainingTask().getUserCodeClassLoader());
		gComparator = null;
	}

	BasicTypeInfo<Long> keyTypeInfo = BasicTypeInfo.LONG_TYPE_INFO;
	ListTypeInfo<BaseRow> valueTypeInfo = new ListTypeInfo<>(inputRowType);
	MapStateDescriptor<Long, List<BaseRow>> mapStateDescriptor = new MapStateDescriptor<>(
			"dataState", keyTypeInfo, valueTypeInfo);
	dataState = getRuntimeContext().getMapState(mapStateDescriptor);

	ValueStateDescriptor<Long> lastTriggeringTsDescriptor = new ValueStateDescriptor<>("lastTriggeringTsState",
			Long.class);
	lastTriggeringTsState = getRuntimeContext().getState(lastTriggeringTsDescriptor);
}
 
Example #14
Source File: FirstValueWithRetractAggFunction.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public GenericRowData createAccumulator() {
	// The accumulator schema:
	// firstValue: T
	// fistOrder: Long
	// valueToOrderMap: RawValueData<MapView<T, List<Long>>>
	// orderToValueMap: RawValueData<MapView<Long, List<T>>>
	GenericRowData acc = new GenericRowData(4);
	acc.setField(0, null);
	acc.setField(1, null);
	acc.setField(2, RawValueData.fromObject(
		new MapView<>(getResultType(), new ListTypeInfo<>(Types.LONG))));
	acc.setField(3, RawValueData.fromObject(
		new MapView<>(Types.LONG, new ListTypeInfo<>(getResultType()))));
	return acc;
}
 
Example #15
Source File: FirstValueWithRetractAggFunction.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public TypeInformation<GenericRowData> getAccumulatorType() {
	LogicalType[] fieldTypes = new LogicalType[] {
			fromTypeInfoToLogicalType(getResultType()),
			new BigIntType(),
			new TypeInformationRawType<>(new MapViewTypeInfo<>(getResultType(), new ListTypeInfo<>(Types.LONG), false, false)),
			new TypeInformationRawType<>(new MapViewTypeInfo<>(Types.LONG, new ListTypeInfo<>(getResultType()), false, false))
	};

	String[] fieldNames = new String[] {
			"firstValue",
			"firstOrder",
			"valueToOrderMapView",
			"orderToValueMapView"
	};

	return (TypeInformation) new RowDataTypeInfo(fieldTypes, fieldNames);
}
 
Example #16
Source File: LastValueWithRetractAggFunction.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public GenericRowData createAccumulator() {
	// The accumulator schema:
	// lastValue: T
	// lastOrder: Long
	// valueToOrderMap: RawValueData<MapView<T, List<Long>>>
	// orderToValueMap: RawValueData<MapView<Long, List<T>>>
	GenericRowData acc = new GenericRowData(4);
	acc.setField(0, null);
	acc.setField(1, null);
	acc.setField(2, RawValueData.fromObject(
			new MapView<>(getResultType(), new ListTypeInfo<>(Types.LONG))));
	acc.setField(3, RawValueData.fromObject(
			new MapView<>(Types.LONG, new ListTypeInfo<>(getResultType()))));
	return acc;
}
 
Example #17
Source File: RowTimeRangeBoundedPrecedingFunction.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
	function.open(new PerKeyStateDataViewStore(getRuntimeContext()));

	output = new JoinedRowData();

	ValueStateDescriptor<Long> lastTriggeringTsDescriptor = new ValueStateDescriptor<Long>(
		"lastTriggeringTsState",
		Types.LONG);
	lastTriggeringTsState = getRuntimeContext().getState(lastTriggeringTsDescriptor);

	RowDataTypeInfo accTypeInfo = new RowDataTypeInfo(accTypes);
	ValueStateDescriptor<RowData> accStateDesc = new ValueStateDescriptor<RowData>("accState", accTypeInfo);
	accState = getRuntimeContext().getState(accStateDesc);

	// input element are all binary row as they are came from network
	RowDataTypeInfo inputType = new RowDataTypeInfo(inputFieldTypes);
	ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
	MapStateDescriptor<Long, List<RowData>> inputStateDesc = new MapStateDescriptor<Long, List<RowData>>(
		"inputState",
		Types.LONG,
		rowListTypeInfo);
	inputState = getRuntimeContext().getMapState(inputStateDesc);

	ValueStateDescriptor<Long> cleanupTsStateDescriptor = new ValueStateDescriptor<>(
		"cleanupTsState",
		Types.LONG
	);
	this.cleanupTsState = getRuntimeContext().getState(cleanupTsStateDescriptor);
}
 
Example #18
Source File: SlidingWindowCheckMapper.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) {
	ValueStateDescriptor<List<Tuple2<Event, Integer>>> previousWindowDescriptor =
		new ValueStateDescriptor<>("eventsSeenSoFar",
			new ListTypeInfo<>(new TupleTypeInfo<>(TypeInformation.of(Event.class), BasicTypeInfo.INT_TYPE_INFO)));

	eventsSeenSoFar = getRuntimeContext().getState(previousWindowDescriptor);

	ValueStateDescriptor<Long> lastSequenceNumberDescriptor =
		new ValueStateDescriptor<>("lastSequenceNumber", BasicTypeInfo.LONG_TYPE_INFO);

	lastSequenceNumber = getRuntimeContext().getState(lastSequenceNumberDescriptor);
}
 
Example #19
Source File: RowTimeRowsBoundedPrecedingFunction.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
	function.open(new PerKeyStateDataViewStore(getRuntimeContext()));

	output = new JoinedRowData();

	ValueStateDescriptor<Long> lastTriggeringTsDescriptor = new ValueStateDescriptor<Long>(
		"lastTriggeringTsState",
		Types.LONG);
	lastTriggeringTsState = getRuntimeContext().getState(lastTriggeringTsDescriptor);

	ValueStateDescriptor<Long> dataCountStateDescriptor = new ValueStateDescriptor<Long>(
		"processedCountState",
		Types.LONG);
	counterState = getRuntimeContext().getState(dataCountStateDescriptor);

	RowDataTypeInfo accTypeInfo = new RowDataTypeInfo(accTypes);
	ValueStateDescriptor<RowData> accStateDesc = new ValueStateDescriptor<RowData>("accState", accTypeInfo);
	accState = getRuntimeContext().getState(accStateDesc);

	// input element are all binary row as they are came from network
	RowDataTypeInfo inputType = new RowDataTypeInfo(inputFieldTypes);
	ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
	MapStateDescriptor<Long, List<RowData>> inputStateDesc = new MapStateDescriptor<Long, List<RowData>>(
		"inputState",
		Types.LONG,
		rowListTypeInfo);
	inputState = getRuntimeContext().getMapState(inputStateDesc);

	initCleanupTimeState("RowTimeBoundedRowsOverCleanupTime");
}
 
Example #20
Source File: AppendOnlyTopNFunction.java    From flink with Apache License 2.0 5 votes vote down vote up
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	int lruCacheSize = Math.max(1, (int) (cacheSize / getDefaultTopNSize()));
	kvSortedMap = new LRUMap<>(lruCacheSize);
	LOG.info("Top{} operator is using LRU caches key-size: {}", getDefaultTopNSize(), lruCacheSize);

	ListTypeInfo<RowData> valueTypeInfo = new ListTypeInfo<>(inputRowType);
	MapStateDescriptor<RowData, List<RowData>> mapStateDescriptor = new MapStateDescriptor<>(
			"data-state-with-append", sortKeyType, valueTypeInfo);
	dataState = getRuntimeContext().getMapState(mapStateDescriptor);

	// metrics
	registerMetric(kvSortedMap.size() * getDefaultTopNSize());
}
 
Example #21
Source File: ProcTimeRowsBoundedPrecedingFunction.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
	function.open(new PerKeyStateDataViewStore(getRuntimeContext()));

	output = new JoinedRowData();

	// input element are all binary row as they are came from network
	RowDataTypeInfo inputType = new RowDataTypeInfo(inputFieldTypes);
	// We keep the elements received in a Map state keyed
	// by the ingestion time in the operator.
	// we also keep counter of processed elements
	// and timestamp of oldest element
	ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
	MapStateDescriptor<Long, List<RowData>> mapStateDescriptor = new MapStateDescriptor<Long, List<RowData>>(
		"inputState", BasicTypeInfo.LONG_TYPE_INFO, rowListTypeInfo);
	inputState = getRuntimeContext().getMapState(mapStateDescriptor);

	RowDataTypeInfo accTypeInfo = new RowDataTypeInfo(accTypes);
	ValueStateDescriptor<RowData> stateDescriptor =
		new ValueStateDescriptor<RowData>("accState", accTypeInfo);
	accState = getRuntimeContext().getState(stateDescriptor);

	ValueStateDescriptor<Long> processedCountDescriptor = new ValueStateDescriptor<Long>(
		"processedCountState",
		Types.LONG);
	counterState = getRuntimeContext().getState(processedCountDescriptor);

	ValueStateDescriptor<Long> smallestTimestampDescriptor = new ValueStateDescriptor<Long>(
		"smallestTSState",
		Types.LONG);
	smallestTsState = getRuntimeContext().getState(smallestTimestampDescriptor);

	initCleanupTimeState("ProcTimeBoundedRowsOverCleanupTime");
}
 
Example #22
Source File: TimeIntervalJoin.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	LOGGER.debug("Instantiating JoinFunction: {} \n\n Code:\n{}", genJoinFunc.getClassName(),
			genJoinFunc.getCode());
	joinFunction = genJoinFunc.newInstance(getRuntimeContext().getUserCodeClassLoader());
	genJoinFunc = null;

	joinCollector = new EmitAwareCollector();

	// Initialize the data caches.
	ListTypeInfo<Tuple2<RowData, Boolean>> leftRowListTypeInfo = new ListTypeInfo<>(
			new TupleTypeInfo<>(leftType, BasicTypeInfo.BOOLEAN_TYPE_INFO));
	MapStateDescriptor<Long, List<Tuple2<RowData, Boolean>>> leftMapStateDescriptor = new MapStateDescriptor<>(
			"IntervalJoinLeftCache",
			BasicTypeInfo.LONG_TYPE_INFO,
			leftRowListTypeInfo);
	leftCache = getRuntimeContext().getMapState(leftMapStateDescriptor);

	ListTypeInfo<Tuple2<RowData, Boolean>> rightRowListTypeInfo = new ListTypeInfo<>(
			new TupleTypeInfo<>(rightType, BasicTypeInfo.BOOLEAN_TYPE_INFO));
	MapStateDescriptor<Long, List<Tuple2<RowData, Boolean>>> rightMapStateDescriptor = new MapStateDescriptor<>(
			"IntervalJoinRightCache",
			BasicTypeInfo.LONG_TYPE_INFO,
			rightRowListTypeInfo);
	rightCache = getRuntimeContext().getMapState(rightMapStateDescriptor);

	// Initialize the timer states.
	ValueStateDescriptor<Long> leftValueStateDescriptor = new ValueStateDescriptor<>(
			"IntervalJoinLeftTimerState",
			Long.class);
	leftTimerState = getRuntimeContext().getState(leftValueStateDescriptor);

	ValueStateDescriptor<Long> rightValueStateDescriptor = new ValueStateDescriptor<>(
			"IntervalJoinRightTimerState",
			Long.class);
	rightTimerState = getRuntimeContext().getState(rightValueStateDescriptor);

	paddingUtil = new OuterJoinPaddingUtil(leftType.getArity(), rightType.getArity());
}
 
Example #23
Source File: SlidingWindowCheckMapper.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) {
	ValueStateDescriptor<List<Tuple2<Event, Integer>>> previousWindowDescriptor =
		new ValueStateDescriptor<>("eventsSeenSoFar",
			new ListTypeInfo<>(new TupleTypeInfo<>(TypeInformation.of(Event.class), BasicTypeInfo.INT_TYPE_INFO)));

	eventsSeenSoFar = getRuntimeContext().getState(previousWindowDescriptor);

	ValueStateDescriptor<Long> lastSequenceNumberDescriptor =
		new ValueStateDescriptor<>("lastSequenceNumber", BasicTypeInfo.LONG_TYPE_INFO);

	lastSequenceNumber = getRuntimeContext().getState(lastSequenceNumberDescriptor);
}
 
Example #24
Source File: AppendOnlyTopNFunction.java    From flink with Apache License 2.0 5 votes vote down vote up
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	int lruCacheSize = Math.max(1, (int) (cacheSize / getDefaultTopNSize()));
	kvSortedMap = new LRUMap<>(lruCacheSize);
	LOG.info("Top{} operator is using LRU caches key-size: {}", getDefaultTopNSize(), lruCacheSize);

	ListTypeInfo<BaseRow> valueTypeInfo = new ListTypeInfo<>(inputRowType);
	MapStateDescriptor<BaseRow, List<BaseRow>> mapStateDescriptor = new MapStateDescriptor<>(
			"data-state-with-append", sortKeyType, valueTypeInfo);
	dataState = getRuntimeContext().getMapState(mapStateDescriptor);

	// metrics
	registerMetric(kvSortedMap.size() * getDefaultTopNSize());
}
 
Example #25
Source File: RowTimeRangeBoundedPrecedingFunction.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
	function.open(new PerKeyStateDataViewStore(getRuntimeContext()));

	output = new JoinedRow();

	ValueStateDescriptor<Long> lastTriggeringTsDescriptor = new ValueStateDescriptor<Long>(
		"lastTriggeringTsState",
		Types.LONG);
	lastTriggeringTsState = getRuntimeContext().getState(lastTriggeringTsDescriptor);

	BaseRowTypeInfo accTypeInfo = new BaseRowTypeInfo(accTypes);
	ValueStateDescriptor<BaseRow> accStateDesc = new ValueStateDescriptor<BaseRow>("accState", accTypeInfo);
	accState = getRuntimeContext().getState(accStateDesc);

	// input element are all binary row as they are came from network
	BaseRowTypeInfo inputType = new BaseRowTypeInfo(inputFieldTypes);
	ListTypeInfo<BaseRow> rowListTypeInfo = new ListTypeInfo<BaseRow>(inputType);
	MapStateDescriptor<Long, List<BaseRow>> inputStateDesc = new MapStateDescriptor<Long, List<BaseRow>>(
		"inputState",
		Types.LONG,
		rowListTypeInfo);
	inputState = getRuntimeContext().getMapState(inputStateDesc);

	initCleanupTimeState("RowTimeBoundedRangeOverCleanupTime");
}
 
Example #26
Source File: RowTimeRowsBoundedPrecedingFunction.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
	function.open(new PerKeyStateDataViewStore(getRuntimeContext()));

	output = new JoinedRow();

	ValueStateDescriptor<Long> lastTriggeringTsDescriptor = new ValueStateDescriptor<Long>(
		"lastTriggeringTsState",
		Types.LONG);
	lastTriggeringTsState = getRuntimeContext().getState(lastTriggeringTsDescriptor);

	ValueStateDescriptor<Long> dataCountStateDescriptor = new ValueStateDescriptor<Long>(
		"processedCountState",
		Types.LONG);
	counterState = getRuntimeContext().getState(dataCountStateDescriptor);

	BaseRowTypeInfo accTypeInfo = new BaseRowTypeInfo(accTypes);
	ValueStateDescriptor<BaseRow> accStateDesc = new ValueStateDescriptor<BaseRow>("accState", accTypeInfo);
	accState = getRuntimeContext().getState(accStateDesc);

	// input element are all binary row as they are came from network
	BaseRowTypeInfo inputType = new BaseRowTypeInfo(inputFieldTypes);
	ListTypeInfo<BaseRow> rowListTypeInfo = new ListTypeInfo<BaseRow>(inputType);
	MapStateDescriptor<Long, List<BaseRow>> inputStateDesc = new MapStateDescriptor<Long, List<BaseRow>>(
		"inputState",
		Types.LONG,
		rowListTypeInfo);
	inputState = getRuntimeContext().getMapState(inputStateDesc);

	initCleanupTimeState("RowTimeBoundedRowsOverCleanupTime");
}
 
Example #27
Source File: ProcTimeRowsBoundedPrecedingFunction.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
	function.open(new PerKeyStateDataViewStore(getRuntimeContext()));

	output = new JoinedRow();

	// input element are all binary row as they are came from network
	BaseRowTypeInfo inputType = new BaseRowTypeInfo(inputFieldTypes);
	// We keep the elements received in a Map state keyed
	// by the ingestion time in the operator.
	// we also keep counter of processed elements
	// and timestamp of oldest element
	ListTypeInfo<BaseRow> rowListTypeInfo = new ListTypeInfo<BaseRow>(inputType);
	MapStateDescriptor<Long, List<BaseRow>> mapStateDescriptor = new MapStateDescriptor<Long, List<BaseRow>>(
		"inputState", BasicTypeInfo.LONG_TYPE_INFO, rowListTypeInfo);
	inputState = getRuntimeContext().getMapState(mapStateDescriptor);

	BaseRowTypeInfo accTypeInfo = new BaseRowTypeInfo(accTypes);
	ValueStateDescriptor<BaseRow> stateDescriptor =
		new ValueStateDescriptor<BaseRow>("accState", accTypeInfo);
	accState = getRuntimeContext().getState(stateDescriptor);

	ValueStateDescriptor<Long> processedCountDescriptor = new ValueStateDescriptor<Long>(
		"processedCountState",
		Types.LONG);
	counterState = getRuntimeContext().getState(processedCountDescriptor);

	ValueStateDescriptor<Long> smallestTimestampDescriptor = new ValueStateDescriptor<Long>(
		"smallestTSState",
		Types.LONG);
	smallestTsState = getRuntimeContext().getState(smallestTimestampDescriptor);

	initCleanupTimeState("ProcTimeBoundedRowsOverCleanupTime");
}
 
Example #28
Source File: TimeBoundedStreamJoin.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	LOGGER.debug("Instantiating JoinFunction: {} \n\n Code:\n{}", genJoinFunc.getClassName(),
			genJoinFunc.getCode());
	joinFunction = genJoinFunc.newInstance(getRuntimeContext().getUserCodeClassLoader());
	genJoinFunc = null;

	joinCollector = new EmitAwareCollector();

	// Initialize the data caches.
	ListTypeInfo<Tuple2<BaseRow, Boolean>> leftRowListTypeInfo = new ListTypeInfo<>(
			new TupleTypeInfo<>(leftType, BasicTypeInfo.BOOLEAN_TYPE_INFO));
	MapStateDescriptor<Long, List<Tuple2<BaseRow, Boolean>>> leftMapStateDescriptor = new MapStateDescriptor<>(
			"WindowJoinLeftCache",
			BasicTypeInfo.LONG_TYPE_INFO,
			leftRowListTypeInfo);
	leftCache = getRuntimeContext().getMapState(leftMapStateDescriptor);

	ListTypeInfo<Tuple2<BaseRow, Boolean>> rightRowListTypeInfo = new ListTypeInfo<>(
			new TupleTypeInfo<>(rightType, BasicTypeInfo.BOOLEAN_TYPE_INFO));
	MapStateDescriptor<Long, List<Tuple2<BaseRow, Boolean>>> rightMapStateDescriptor = new MapStateDescriptor<>(
			"WindowJoinRightCache",
			BasicTypeInfo.LONG_TYPE_INFO,
			rightRowListTypeInfo);
	rightCache = getRuntimeContext().getMapState(rightMapStateDescriptor);

	// Initialize the timer states.
	ValueStateDescriptor<Long> leftValueStateDescriptor = new ValueStateDescriptor<>(
			"WindowJoinLeftTimerState",
			Long.class);
	leftTimerState = getRuntimeContext().getState(leftValueStateDescriptor);

	ValueStateDescriptor<Long> rightValueStateDescriptor = new ValueStateDescriptor<>(
			"WindowJoinRightTimerState",
			Long.class);
	rightTimerState = getRuntimeContext().getState(rightValueStateDescriptor);

	paddingUtil = new OuterJoinPaddingUtil(leftType.getArity(), rightType.getArity());
}
 
Example #29
Source File: SlidingWindowCheckMapper.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) {
	ValueStateDescriptor<List<Tuple2<Event, Integer>>> previousWindowDescriptor =
		new ValueStateDescriptor<>("eventsSeenSoFar",
			new ListTypeInfo<>(new TupleTypeInfo<>(TypeInformation.of(Event.class), BasicTypeInfo.INT_TYPE_INFO)));

	eventsSeenSoFar = getRuntimeContext().getState(previousWindowDescriptor);

	ValueStateDescriptor<Long> lastSequenceNumberDescriptor =
		new ValueStateDescriptor<>("lastSequenceNumber", BasicTypeInfo.LONG_TYPE_INFO);

	lastSequenceNumber = getRuntimeContext().getState(lastSequenceNumberDescriptor);
}
 
Example #30
Source File: ListStateListReader.java    From bravo with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
public ListStateListReader(String stateName, TypeInformation<K> outKeyType, TypeInformation<V> outValueType) {
	super(stateName, outKeyType, outValueType,
			new TupleTypeInfo(Tuple2.class, outKeyType, new ListTypeInfo<>(outValueType)));
}