Java Code Examples for org.apache.flink.streaming.runtime.streamstatus.StreamStatus#ACTIVE

The following examples show how to use org.apache.flink.streaming.runtime.streamstatus.StreamStatus#ACTIVE . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StreamMultipleInputProcessor.java    From flink with Apache License 2.0 5 votes vote down vote up 
public StreamMultipleInputProcessor(
		CheckpointedInputGate[] checkpointedInputGates,
		TypeSerializer<?>[] inputSerializers,
		IOManager ioManager,
		StreamStatusMaintainer streamStatusMaintainer,
		MultipleInputStreamOperator<?> streamOperator,
		MultipleInputSelectionHandler inputSelectionHandler,
		WatermarkGauge[] inputWatermarkGauges,
		OperatorChain<?, ?> operatorChain,
		Counter numRecordsIn) {

	this.inputSelectionHandler = checkNotNull(inputSelectionHandler);

	List<Input> inputs = streamOperator.getInputs();
	int inputsCount = inputs.size();

	this.inputProcessors = new InputProcessor[inputsCount];
	this.streamStatuses = new StreamStatus[inputsCount];
	this.numRecordsIn = numRecordsIn;

	for (int i = 0; i < inputsCount; i++) {
		streamStatuses[i] = StreamStatus.ACTIVE;
		StreamTaskNetworkOutput dataOutput = new StreamTaskNetworkOutput<>(
			inputs.get(i),
			streamStatusMaintainer,
			inputWatermarkGauges[i],
			i);

		inputProcessors[i] = new InputProcessor(
			dataOutput,
			new StreamTaskNetworkInput<>(
				checkpointedInputGates[i],
				inputSerializers[i],
				ioManager,
				new StatusWatermarkValve(checkpointedInputGates[i].getNumberOfInputChannels(), dataOutput),
				i));
	}

	this.operatorChain = checkNotNull(operatorChain);
}
Example 2
Source File: StreamTwoInputProcessor.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
public StreamTwoInputProcessor(
		Collection<InputGate> inputGates1,
		Collection<InputGate> inputGates2,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		TwoInputStreamTask<IN1, IN2, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge input1WatermarkGauge,
		WatermarkGauge input2WatermarkGauge) throws IOException {

	final InputGate inputGate = InputGateUtil.createInputGate(inputGates1, inputGates2);

	this.barrierHandler = InputProcessorUtil.createCheckpointBarrierHandler(
		checkpointedTask, checkpointMode, ioManager, inputGate, taskManagerConfig);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN1> ser1 = new StreamElementSerializer<>(inputSerializer1);
	this.deserializationDelegate1 = new NonReusingDeserializationDelegate<>(ser1);

	StreamElementSerializer<IN2> ser2 = new StreamElementSerializer<>(inputSerializer2);
	this.deserializationDelegate2 = new NonReusingDeserializationDelegate<>(ser2);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	// determine which unioned channels belong to input 1 and which belong to input 2
	int numInputChannels1 = 0;
	for (InputGate gate: inputGates1) {
		numInputChannels1 += gate.getNumberOfInputChannels();
	}

	this.numInputChannels1 = numInputChannels1;
	this.numInputChannels2 = inputGate.getNumberOfInputChannels() - numInputChannels1;

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve1 = new StatusWatermarkValve(numInputChannels1, new ForwardingValveOutputHandler1(streamOperator, lock));
	this.statusWatermarkValve2 = new StatusWatermarkValve(numInputChannels2, new ForwardingValveOutputHandler2(streamOperator, lock));

	this.input1WatermarkGauge = input1WatermarkGauge;
	this.input2WatermarkGauge = input2WatermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);
}
Example 3
Source File: StreamTwoInputSelectableProcessor.java    From flink with Apache License 2.0 4 votes vote down vote up 
public StreamTwoInputSelectableProcessor(
	Collection<InputGate> inputGates1,
	Collection<InputGate> inputGates2,
	TypeSerializer<IN1> inputSerializer1,
	TypeSerializer<IN2> inputSerializer2,
	StreamTask<?, ?> streamTask,
	CheckpointingMode checkpointingMode,
	Object lock,
	IOManager ioManager,
	Configuration taskManagerConfig,
	StreamStatusMaintainer streamStatusMaintainer,
	TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
	WatermarkGauge input1WatermarkGauge,
	WatermarkGauge input2WatermarkGauge,
	String taskName,
	OperatorChain<?, ?> operatorChain) throws IOException {

	checkState(streamOperator instanceof InputSelectable);

	this.streamOperator = checkNotNull(streamOperator);
	this.inputSelector = (InputSelectable) streamOperator;

	this.lock = checkNotNull(lock);

	InputGate unionedInputGate1 = InputGateUtil.createInputGate(inputGates1.toArray(new InputGate[0]));
	InputGate unionedInputGate2 = InputGateUtil.createInputGate(inputGates2.toArray(new InputGate[0]));

	// create a Input instance for each input
	CheckpointedInputGate[] checkpointedInputGates = InputProcessorUtil.createCheckpointedInputGatePair(
		streamTask,
		checkpointingMode,
		ioManager,
		unionedInputGate1,
		unionedInputGate2,
		taskManagerConfig,
		taskName);
	checkState(checkpointedInputGates.length == 2);
	this.input1 = new StreamTaskNetworkInput(checkpointedInputGates[0], inputSerializer1, ioManager, 0);
	this.input2 = new StreamTaskNetworkInput(checkpointedInputGates[1], inputSerializer2, ioManager, 1);

	this.statusWatermarkValve1 = new StatusWatermarkValve(
		unionedInputGate1.getNumberOfInputChannels(),
		new ForwardingValveOutputHandler(streamOperator, lock, streamStatusMaintainer, input1WatermarkGauge, 0));
	this.statusWatermarkValve2 = new StatusWatermarkValve(
		unionedInputGate2.getNumberOfInputChannels(),
		new ForwardingValveOutputHandler(streamOperator, lock, streamStatusMaintainer, input2WatermarkGauge, 1));

	this.operatorChain = checkNotNull(operatorChain);

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.availableInputsMask = (int) new InputSelection.Builder().select(1).select(2).build().getInputMask();

	this.lastReadInputIndex = 1; // always try to read from the first input

	this.isPrepared = false;
}
Example 4
Source File: StreamTwoInputProcessor.java    From flink with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
public StreamTwoInputProcessor(
		Collection<InputGate> inputGates1,
		Collection<InputGate> inputGates2,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		TwoInputStreamTask<IN1, IN2, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge input1WatermarkGauge,
		WatermarkGauge input2WatermarkGauge,
		String taskName,
		OperatorChain<?, ?> operatorChain) throws IOException {

	final InputGate inputGate = InputGateUtil.createInputGate(inputGates1, inputGates2);

	this.barrierHandler = InputProcessorUtil.createCheckpointedInputGate(
		checkpointedTask,
		checkpointMode,
		ioManager,
		inputGate,
		taskManagerConfig,
		taskName);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN1> ser1 = new StreamElementSerializer<>(inputSerializer1);
	this.deserializationDelegate1 = new NonReusingDeserializationDelegate<>(ser1);

	StreamElementSerializer<IN2> ser2 = new StreamElementSerializer<>(inputSerializer2);
	this.deserializationDelegate2 = new NonReusingDeserializationDelegate<>(ser2);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	// determine which unioned channels belong to input 1 and which belong to input 2
	int numInputChannels1 = 0;
	for (InputGate gate: inputGates1) {
		numInputChannels1 += gate.getNumberOfInputChannels();
	}

	this.numInputChannels1 = numInputChannels1;
	this.numInputChannels2 = inputGate.getNumberOfInputChannels() - numInputChannels1;

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve1 = new StatusWatermarkValve(numInputChannels1, new ForwardingValveOutputHandler1(streamOperator, lock));
	this.statusWatermarkValve2 = new StatusWatermarkValve(numInputChannels2, new ForwardingValveOutputHandler2(streamOperator, lock));

	this.input1WatermarkGauge = input1WatermarkGauge;
	this.input2WatermarkGauge = input2WatermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);

	this.operatorChain = checkNotNull(operatorChain);

	this.finishedChannels1 = new BitSet();
	this.finishedChannels2 = new BitSet();
}