Java Code Examples for org.apache.flink.runtime.io.network.partition.consumer.InputGate#getNumberOfInputChannels()

/**
 * Creates a new AbstractRecordReader that de-serializes records from the given input gate and
 * can spill partial records to disk, if they grow large.
 *
 * @param inputGate The input gate to read from.
 * @param tmpDirectories The temp directories. USed for spilling if the reader concurrently
 *                       reconstructs multiple large records.
 */
@SuppressWarnings("unchecked")
protected AbstractRecordReader(InputGate inputGate, String[] tmpDirectories) {
	super(inputGate);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];
	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<T>(tmpDirectories);
	}
}
 

/**
 * Creates a new AbstractRecordReader that de-serializes records from the given input gate and
 * can spill partial records to disk, if they grow large.
 *
 * @param inputGate The input gate to read from.
 * @param tmpDirectories The temp directories. USed for spilling if the reader concurrently
 *                       reconstructs multiple large records.
 */
@SuppressWarnings("unchecked")
protected AbstractRecordReader(InputGate inputGate, String[] tmpDirectories) {
	super(inputGate);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];
	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<T>(tmpDirectories);
	}
}
 

public CheckpointedInputGate(
		InputGate inputGate,
		BufferStorage bufferStorage,
		String taskName,
		@Nullable AbstractInvokable toNotifyOnCheckpoint) {
	this(
		inputGate,
		bufferStorage,
		new CheckpointBarrierAligner(
			inputGate.getNumberOfInputChannels(),
			taskName,
			toNotifyOnCheckpoint)
	);
}
 

/**
 * @return a pair of {@link CheckpointedInputGate} created for two corresponding
 * {@link InputGate}s supplied as parameters.
 */
public static CheckpointedInputGate[] createCheckpointedInputGatePair(
		AbstractInvokable toNotifyOnCheckpoint,
		CheckpointingMode checkpointMode,
		IOManager ioManager,
		InputGate inputGate1,
		InputGate inputGate2,
		Configuration taskManagerConfig,
		String taskName) throws IOException {

	int pageSize = ConfigurationParserUtils.getPageSize(taskManagerConfig);

	BufferStorage mainBufferStorage1 = createBufferStorage(
		checkpointMode, ioManager, pageSize, taskManagerConfig, taskName);
	BufferStorage mainBufferStorage2 = createBufferStorage(
		checkpointMode, ioManager, pageSize, taskManagerConfig, taskName);
	checkState(mainBufferStorage1.getMaxBufferedBytes() == mainBufferStorage2.getMaxBufferedBytes());

	BufferStorage linkedBufferStorage1 = new LinkedBufferStorage(
		mainBufferStorage1,
		mainBufferStorage2,
		mainBufferStorage1.getMaxBufferedBytes());
	BufferStorage linkedBufferStorage2 = new LinkedBufferStorage(
		mainBufferStorage2,
		mainBufferStorage1,
		mainBufferStorage1.getMaxBufferedBytes());

	CheckpointBarrierHandler barrierHandler = createCheckpointBarrierHandler(
		checkpointMode,
		inputGate1.getNumberOfInputChannels() + inputGate2.getNumberOfInputChannels(),
		taskName,
		toNotifyOnCheckpoint);
	return new CheckpointedInputGate[] {
		new CheckpointedInputGate(inputGate1, linkedBufferStorage1, barrierHandler),
		new CheckpointedInputGate(inputGate2, linkedBufferStorage2, barrierHandler, inputGate1.getNumberOfInputChannels())
	};
}
 

@SuppressWarnings("unchecked")
public StreamInputProcessor(
		InputGate[] inputGates,
		TypeSerializer<IN> inputSerializer,
		StreamTask<?, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		OneInputStreamOperator<IN, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge watermarkGauge) throws IOException {

	InputGate inputGate = InputGateUtil.createInputGate(inputGates);

	this.barrierHandler = InputProcessorUtil.createCheckpointBarrierHandler(
		checkpointedTask, checkpointMode, ioManager, inputGate, taskManagerConfig);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN> ser = new StreamElementSerializer<>(inputSerializer);
	this.deserializationDelegate = new NonReusingDeserializationDelegate<>(ser);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	this.numInputChannels = inputGate.getNumberOfInputChannels();

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve = new StatusWatermarkValve(
			numInputChannels,
			new ForwardingValveOutputHandler(streamOperator, lock));

	this.watermarkGauge = watermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);
}
 

@SuppressWarnings("unchecked")
public StreamTwoInputProcessor(
		Collection<InputGate> inputGates1,
		Collection<InputGate> inputGates2,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		TwoInputStreamTask<IN1, IN2, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge input1WatermarkGauge,
		WatermarkGauge input2WatermarkGauge) throws IOException {

	final InputGate inputGate = InputGateUtil.createInputGate(inputGates1, inputGates2);

	this.barrierHandler = InputProcessorUtil.createCheckpointBarrierHandler(
		checkpointedTask, checkpointMode, ioManager, inputGate, taskManagerConfig);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN1> ser1 = new StreamElementSerializer<>(inputSerializer1);
	this.deserializationDelegate1 = new NonReusingDeserializationDelegate<>(ser1);

	StreamElementSerializer<IN2> ser2 = new StreamElementSerializer<>(inputSerializer2);
	this.deserializationDelegate2 = new NonReusingDeserializationDelegate<>(ser2);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	// determine which unioned channels belong to input 1 and which belong to input 2
	int numInputChannels1 = 0;
	for (InputGate gate: inputGates1) {
		numInputChannels1 += gate.getNumberOfInputChannels();
	}

	this.numInputChannels1 = numInputChannels1;
	this.numInputChannels2 = inputGate.getNumberOfInputChannels() - numInputChannels1;

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve1 = new StatusWatermarkValve(numInputChannels1, new ForwardingValveOutputHandler1(streamOperator, lock));
	this.statusWatermarkValve2 = new StatusWatermarkValve(numInputChannels2, new ForwardingValveOutputHandler2(streamOperator, lock));

	this.input1WatermarkGauge = input1WatermarkGauge;
	this.input2WatermarkGauge = input2WatermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);
}
 

public BarrierTracker(InputGate inputGate) {
	this.inputGate = inputGate;
	this.totalNumberOfInputChannels = inputGate.getNumberOfInputChannels();
	this.pendingCheckpoints = new ArrayDeque<>();
}
 

@SuppressWarnings("unchecked")
public StreamOneInputProcessor(
		InputGate[] inputGates,
		TypeSerializer<IN> inputSerializer,
		StreamTask<?, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		OneInputStreamOperator<IN, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge watermarkGauge,
		String taskName,
		OperatorChain<?, ?> operatorChain) throws IOException {

	InputGate inputGate = InputGateUtil.createInputGate(inputGates);

	CheckpointedInputGate barrierHandler = InputProcessorUtil.createCheckpointedInputGate(
		checkpointedTask,
		checkpointMode,
		ioManager,
		inputGate,
		taskManagerConfig,
		taskName);
	this.input = new StreamTaskNetworkInput(barrierHandler, inputSerializer, ioManager, 0);

	this.lock = checkNotNull(lock);

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve = new StatusWatermarkValve(
		inputGate.getNumberOfInputChannels(),
		new ForwardingValveOutputHandler(streamOperator, lock));

	this.watermarkGauge = watermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);

	this.operatorChain = checkNotNull(operatorChain);
}
 

public StreamTwoInputSelectableProcessor(
	Collection<InputGate> inputGates1,
	Collection<InputGate> inputGates2,
	TypeSerializer<IN1> inputSerializer1,
	TypeSerializer<IN2> inputSerializer2,
	StreamTask<?, ?> streamTask,
	CheckpointingMode checkpointingMode,
	Object lock,
	IOManager ioManager,
	Configuration taskManagerConfig,
	StreamStatusMaintainer streamStatusMaintainer,
	TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
	WatermarkGauge input1WatermarkGauge,
	WatermarkGauge input2WatermarkGauge,
	String taskName,
	OperatorChain<?, ?> operatorChain) throws IOException {

	checkState(streamOperator instanceof InputSelectable);

	this.streamOperator = checkNotNull(streamOperator);
	this.inputSelector = (InputSelectable) streamOperator;

	this.lock = checkNotNull(lock);

	InputGate unionedInputGate1 = InputGateUtil.createInputGate(inputGates1.toArray(new InputGate[0]));
	InputGate unionedInputGate2 = InputGateUtil.createInputGate(inputGates2.toArray(new InputGate[0]));

	// create a Input instance for each input
	CheckpointedInputGate[] checkpointedInputGates = InputProcessorUtil.createCheckpointedInputGatePair(
		streamTask,
		checkpointingMode,
		ioManager,
		unionedInputGate1,
		unionedInputGate2,
		taskManagerConfig,
		taskName);
	checkState(checkpointedInputGates.length == 2);
	this.input1 = new StreamTaskNetworkInput(checkpointedInputGates[0], inputSerializer1, ioManager, 0);
	this.input2 = new StreamTaskNetworkInput(checkpointedInputGates[1], inputSerializer2, ioManager, 1);

	this.statusWatermarkValve1 = new StatusWatermarkValve(
		unionedInputGate1.getNumberOfInputChannels(),
		new ForwardingValveOutputHandler(streamOperator, lock, streamStatusMaintainer, input1WatermarkGauge, 0));
	this.statusWatermarkValve2 = new StatusWatermarkValve(
		unionedInputGate2.getNumberOfInputChannels(),
		new ForwardingValveOutputHandler(streamOperator, lock, streamStatusMaintainer, input2WatermarkGauge, 1));

	this.operatorChain = checkNotNull(operatorChain);

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.availableInputsMask = (int) new InputSelection.Builder().select(1).select(2).build().getInputMask();

	this.lastReadInputIndex = 1; // always try to read from the first input

	this.isPrepared = false;
}
 

@SuppressWarnings("unchecked")
public StreamTwoInputProcessor(
		Collection<InputGate> inputGates1,
		Collection<InputGate> inputGates2,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		TwoInputStreamTask<IN1, IN2, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge input1WatermarkGauge,
		WatermarkGauge input2WatermarkGauge,
		String taskName,
		OperatorChain<?, ?> operatorChain) throws IOException {

	final InputGate inputGate = InputGateUtil.createInputGate(inputGates1, inputGates2);

	this.barrierHandler = InputProcessorUtil.createCheckpointedInputGate(
		checkpointedTask,
		checkpointMode,
		ioManager,
		inputGate,
		taskManagerConfig,
		taskName);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN1> ser1 = new StreamElementSerializer<>(inputSerializer1);
	this.deserializationDelegate1 = new NonReusingDeserializationDelegate<>(ser1);

	StreamElementSerializer<IN2> ser2 = new StreamElementSerializer<>(inputSerializer2);
	this.deserializationDelegate2 = new NonReusingDeserializationDelegate<>(ser2);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	// determine which unioned channels belong to input 1 and which belong to input 2
	int numInputChannels1 = 0;
	for (InputGate gate: inputGates1) {
		numInputChannels1 += gate.getNumberOfInputChannels();
	}

	this.numInputChannels1 = numInputChannels1;
	this.numInputChannels2 = inputGate.getNumberOfInputChannels() - numInputChannels1;

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve1 = new StatusWatermarkValve(numInputChannels1, new ForwardingValveOutputHandler1(streamOperator, lock));
	this.statusWatermarkValve2 = new StatusWatermarkValve(numInputChannels2, new ForwardingValveOutputHandler2(streamOperator, lock));

	this.input1WatermarkGauge = input1WatermarkGauge;
	this.input2WatermarkGauge = input2WatermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);

	this.operatorChain = checkNotNull(operatorChain);

	this.finishedChannels1 = new BitSet();
	this.finishedChannels2 = new BitSet();
}
 

/**
 * Creates a new checkpoint stream aligner.
 *
 * <p>The aligner will allow only alignments that buffer up to the given number of bytes.
 * When that number is exceeded, it will stop the alignment and notify the task that the
 * checkpoint has been cancelled.
 *
 * @param inputGate The input gate to draw the buffers and events from.
 * @param bufferBlocker The buffer blocker to hold the buffers and events for channels with barrier.
 * @param maxBufferedBytes The maximum bytes to be buffered before the checkpoint aborts.
 *
 * @throws IOException Thrown, when the spilling to temp files cannot be initialized.
 */
public BarrierBuffer(InputGate inputGate, BufferBlocker bufferBlocker, long maxBufferedBytes)
		throws IOException {
	checkArgument(maxBufferedBytes == -1 || maxBufferedBytes > 0);

	this.inputGate = inputGate;
	this.maxBufferedBytes = maxBufferedBytes;
	this.totalNumberOfInputChannels = inputGate.getNumberOfInputChannels();
	this.blockedChannels = new boolean[this.totalNumberOfInputChannels];

	this.bufferBlocker = checkNotNull(bufferBlocker);
	this.queuedBuffered = new ArrayDeque<BufferOrEventSequence>();
}