org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup Java Examples

The following examples show how to use org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: InputProcessorUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public static CheckpointedInputGate createCheckpointedInputGate(
		AbstractInvokable toNotifyOnCheckpoint,
		StreamConfig config,
		SubtaskCheckpointCoordinator checkpointCoordinator,
		IndexedInputGate[] inputGates,
		TaskIOMetricGroup taskIOMetricGroup,
		String taskName) {
	CheckpointedInputGate[] checkpointedInputGates = createCheckpointedMultipleInputGate(
		toNotifyOnCheckpoint,
		config,
		checkpointCoordinator,
		taskIOMetricGroup,
		taskName,
		Arrays.asList(inputGates));
	return Iterables.getOnlyElement(Arrays.asList(checkpointedInputGates));
}
 
Example #2
Source File: InputGateFairnessTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public FairnessVerifyingInputGate(
		String owningTaskName,
		JobID jobId,
		IntermediateDataSetID consumedResultId,
		int consumedSubpartitionIndex,
		int numberOfInputChannels,
		TaskActions taskActions,
		TaskIOMetricGroup metrics,
		boolean isCreditBased) {

	super(owningTaskName, jobId, consumedResultId, ResultPartitionType.PIPELINED,
		consumedSubpartitionIndex,
			numberOfInputChannels, taskActions, metrics, isCreditBased);

	try {
		Field f = SingleInputGate.class.getDeclaredField("inputChannelsWithData");
		f.setAccessible(true);
		channelsWithData = (ArrayDeque<InputChannel>) f.get(this);
	}
	catch (Exception e) {
		throw new RuntimeException(e);
	}

	this.uniquenessChecker = new HashSet<>();
}
 
Example #3
Source File: UnknownInputChannel.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public UnknownInputChannel(
		SingleInputGate gate,
		int channelIndex,
		ResultPartitionID partitionId,
		ResultPartitionManager partitionManager,
		TaskEventDispatcher taskEventDispatcher,
		ConnectionManager connectionManager,
		int initialBackoff,
		int maxBackoff,
		TaskIOMetricGroup metrics) {

	super(gate, channelIndex, partitionId, initialBackoff, maxBackoff, null, null);

	this.partitionManager = checkNotNull(partitionManager);
	this.taskEventDispatcher = checkNotNull(taskEventDispatcher);
	this.connectionManager = checkNotNull(connectionManager);
	this.metrics = checkNotNull(metrics);
	this.initialBackoff = initialBackoff;
	this.maxBackoff = maxBackoff;
}
 
Example #4
Source File: LocalInputChannel.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public LocalInputChannel(
	SingleInputGate inputGate,
	int channelIndex,
	ResultPartitionID partitionId,
	ResultPartitionManager partitionManager,
	TaskEventDispatcher taskEventDispatcher,
	TaskIOMetricGroup metrics) {

	this(inputGate, channelIndex, partitionId, partitionManager, taskEventDispatcher,
		0, 0, metrics);
}
 
Example #5
Source File: InputProcessorUtil.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * @return a pair of {@link CheckpointedInputGate} created for two corresponding
 * {@link InputGate}s supplied as parameters.
 */
public static CheckpointedInputGate[] createCheckpointedMultipleInputGate(
		AbstractInvokable toNotifyOnCheckpoint,
		StreamConfig config,
		SubtaskCheckpointCoordinator checkpointCoordinator,
		TaskIOMetricGroup taskIOMetricGroup,
		String taskName,
		List<IndexedInputGate>... inputGates) {

	IndexedInputGate[] sortedInputGates = Arrays.stream(inputGates)
		.flatMap(Collection::stream)
		.sorted(Comparator.comparing(IndexedInputGate::getGateIndex))
		.toArray(IndexedInputGate[]::new);
	CheckpointBarrierHandler barrierHandler = createCheckpointBarrierHandler(
		config,
		sortedInputGates,
		checkpointCoordinator,
		taskName,
		toNotifyOnCheckpoint);
	registerCheckpointMetrics(taskIOMetricGroup, barrierHandler);

	InputGate[] unionedInputGates = Arrays.stream(inputGates)
		.map(InputGateUtil::createInputGate)
		.toArray(InputGate[]::new);
	barrierHandler.getBufferReceivedListener().ifPresent(listener -> {
		for (final InputGate inputGate : unionedInputGates) {
			inputGate.registerBufferReceivedListener(listener);
		}
	});

	return Arrays.stream(unionedInputGates)
		.map(unionedInputGate -> new CheckpointedInputGate(unionedInputGate, barrierHandler))
		.toArray(CheckpointedInputGate[]::new);
}
 
Example #6
Source File: SingleInputGate.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public SingleInputGate(
	String owningTaskName,
	JobID jobId,
	IntermediateDataSetID consumedResultId,
	final ResultPartitionType consumedPartitionType,
	int consumedSubpartitionIndex,
	int numberOfInputChannels,
	TaskActions taskActions,
	TaskIOMetricGroup metrics,
	boolean isCreditBased) {

	this.owningTaskName = checkNotNull(owningTaskName);
	this.jobId = checkNotNull(jobId);

	this.consumedResultId = checkNotNull(consumedResultId);
	this.consumedPartitionType = checkNotNull(consumedPartitionType);

	checkArgument(consumedSubpartitionIndex >= 0);
	this.consumedSubpartitionIndex = consumedSubpartitionIndex;

	checkArgument(numberOfInputChannels > 0);
	this.numberOfInputChannels = numberOfInputChannels;

	this.inputChannels = new HashMap<>(numberOfInputChannels);
	this.channelsWithEndOfPartitionEvents = new BitSet(numberOfInputChannels);
	this.enqueuedInputChannelsWithData = new BitSet(numberOfInputChannels);

	this.taskActions = checkNotNull(taskActions);
	this.isCreditBased = isCreditBased;
}
 
Example #7
Source File: RemoteInputChannel.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public RemoteInputChannel(
	SingleInputGate inputGate,
	int channelIndex,
	ResultPartitionID partitionId,
	ConnectionID connectionId,
	ConnectionManager connectionManager,
	int initialBackOff,
	int maxBackoff,
	TaskIOMetricGroup metrics) {

	super(inputGate, channelIndex, partitionId, initialBackOff, maxBackoff, metrics.getNumBytesInRemoteCounter(), metrics.getNumBuffersInRemoteCounter());

	this.connectionId = checkNotNull(connectionId);
	this.connectionManager = checkNotNull(connectionManager);
}
 
Example #8
Source File: RemoteInputChannel.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public RemoteInputChannel(
	SingleInputGate inputGate,
	int channelIndex,
	ResultPartitionID partitionId,
	ConnectionID connectionId,
	ConnectionManager connectionManager,
	TaskIOMetricGroup metrics) {

	this(inputGate, channelIndex, partitionId, connectionId, connectionManager, 0, 0, metrics);
}
 
Example #9
Source File: LocalInputChannel.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public LocalInputChannel(
	SingleInputGate inputGate,
	int channelIndex,
	ResultPartitionID partitionId,
	ResultPartitionManager partitionManager,
	TaskEventDispatcher taskEventDispatcher,
	int initialBackoff,
	int maxBackoff,
	TaskIOMetricGroup metrics) {

	super(inputGate, channelIndex, partitionId, initialBackoff, maxBackoff, metrics.getNumBytesInLocalCounter(), metrics.getNumBuffersInLocalCounter());

	this.partitionManager = checkNotNull(partitionManager);
	this.taskEventDispatcher = checkNotNull(taskEventDispatcher);
}
 
Example #10
Source File: SingleInputGate.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Creates an input gate and all of its input channels.
 */
public static SingleInputGate create(
	String owningTaskName,
	JobID jobId,
	ExecutionAttemptID executionId,
	InputGateDeploymentDescriptor igdd,
	NetworkEnvironment networkEnvironment,
	TaskActions taskActions,
	TaskIOMetricGroup metrics) {

	final IntermediateDataSetID consumedResultId = checkNotNull(igdd.getConsumedResultId());
	final ResultPartitionType consumedPartitionType = checkNotNull(igdd.getConsumedPartitionType());

	final int consumedSubpartitionIndex = igdd.getConsumedSubpartitionIndex();
	checkArgument(consumedSubpartitionIndex >= 0);

	final InputChannelDeploymentDescriptor[] icdd = checkNotNull(igdd.getInputChannelDeploymentDescriptors());

	final SingleInputGate inputGate = new SingleInputGate(
		owningTaskName, jobId, consumedResultId, consumedPartitionType, consumedSubpartitionIndex,
		icdd.length, taskActions, metrics, networkEnvironment.isCreditBased());

	// Create the input channels. There is one input channel for each consumed partition.
	final InputChannel[] inputChannels = new InputChannel[icdd.length];

	int numLocalChannels = 0;
	int numRemoteChannels = 0;
	int numUnknownChannels = 0;

	for (int i = 0; i < inputChannels.length; i++) {
		final ResultPartitionID partitionId = icdd[i].getConsumedPartitionId();
		final ResultPartitionLocation partitionLocation = icdd[i].getConsumedPartitionLocation();

		if (partitionLocation.isLocal()) {
			inputChannels[i] = new LocalInputChannel(inputGate, i, partitionId,
				networkEnvironment.getResultPartitionManager(),
				networkEnvironment.getTaskEventDispatcher(),
				networkEnvironment.getPartitionRequestInitialBackoff(),
				networkEnvironment.getPartitionRequestMaxBackoff(),
				metrics
			);

			numLocalChannels++;
		}
		else if (partitionLocation.isRemote()) {
			inputChannels[i] = new RemoteInputChannel(inputGate, i, partitionId,
				partitionLocation.getConnectionId(),
				networkEnvironment.getConnectionManager(),
				networkEnvironment.getPartitionRequestInitialBackoff(),
				networkEnvironment.getPartitionRequestMaxBackoff(),
				metrics
			);

			numRemoteChannels++;
		}
		else if (partitionLocation.isUnknown()) {
			inputChannels[i] = new UnknownInputChannel(inputGate, i, partitionId,
				networkEnvironment.getResultPartitionManager(),
				networkEnvironment.getTaskEventDispatcher(),
				networkEnvironment.getConnectionManager(),
				networkEnvironment.getPartitionRequestInitialBackoff(),
				networkEnvironment.getPartitionRequestMaxBackoff(),
				metrics
			);

			numUnknownChannels++;
		}
		else {
			throw new IllegalStateException("Unexpected partition location.");
		}

		inputGate.setInputChannel(partitionId.getPartitionId(), inputChannels[i]);
	}

	LOG.debug("{}: Created {} input channels (local: {}, remote: {}, unknown: {}).",
		owningTaskName,
		inputChannels.length,
		numLocalChannels,
		numRemoteChannels,
		numUnknownChannels);

	return inputGate;
}
 
Example #11
Source File: TaskAsyncCallTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
private Task createTask(Class<? extends AbstractInvokable> invokableClass) throws Exception {
	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(new TestUserCodeClassLoader());

	ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);
	TaskEventDispatcher taskEventDispatcher = mock(TaskEventDispatcher.class);
	NetworkEnvironment networkEnvironment = mock(NetworkEnvironment.class);
	when(networkEnvironment.getResultPartitionManager()).thenReturn(partitionManager);
	when(networkEnvironment.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);
	when(networkEnvironment.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class)))
			.thenReturn(mock(TaskKvStateRegistry.class));
	when(networkEnvironment.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

	TaskMetricGroup taskMetricGroup = mock(TaskMetricGroup.class);
	when(taskMetricGroup.getIOMetricGroup()).thenReturn(mock(TaskIOMetricGroup.class));

	JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Job Name",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		invokableClass.getName(),
		new Configuration());

	return new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		networkEnvironment,
		mock(BroadcastVariableManager.class),
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		libCache,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(),
		taskMetricGroup,
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 
Example #12
Source File: TaskTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
private Task build() throws Exception {
	final JobID jobId = new JobID();
	final JobVertexID jobVertexId = new JobVertexID();
	final ExecutionAttemptID executionAttemptId = new ExecutionAttemptID();

	final SerializedValue<ExecutionConfig> serializedExecutionConfig = new SerializedValue<>(executionConfig);

	final JobInformation jobInformation = new JobInformation(
		jobId,
		"Test Job",
		serializedExecutionConfig,
		new Configuration(),
		requiredJarFileBlobKeys,
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		jobVertexId,
		"Test Task",
		1,
		1,
		invokable.getName(),
		new Configuration());

	final BlobCacheService blobCacheService = new BlobCacheService(
		mock(PermanentBlobCache.class),
		mock(TransientBlobCache.class));

	final TaskMetricGroup taskMetricGroup = mock(TaskMetricGroup.class);
	when(taskMetricGroup.getIOMetricGroup()).thenReturn(mock(TaskIOMetricGroup.class));

	return new Task(
		jobInformation,
		taskInformation,
		executionAttemptId,
		new AllocationID(),
		0,
		0,
		Collections.emptyList(),
		Collections.emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		networkEnvironment,
		mock(BroadcastVariableManager.class),
		new TestTaskStateManager(),
		taskManagerActions,
		new MockInputSplitProvider(),
		new TestCheckpointResponder(),
		new TestGlobalAggregateManager(),
		blobCacheService,
		libraryCacheManager,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(taskManagerConfig),
		taskMetricGroup,
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 
Example #13
Source File: StreamInputProcessor.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public StreamInputProcessor(
		InputGate[] inputGates,
		TypeSerializer<IN> inputSerializer,
		StreamTask<?, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		OneInputStreamOperator<IN, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge watermarkGauge) throws IOException {

	InputGate inputGate = InputGateUtil.createInputGate(inputGates);

	this.barrierHandler = InputProcessorUtil.createCheckpointBarrierHandler(
		checkpointedTask, checkpointMode, ioManager, inputGate, taskManagerConfig);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN> ser = new StreamElementSerializer<>(inputSerializer);
	this.deserializationDelegate = new NonReusingDeserializationDelegate<>(ser);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	this.numInputChannels = inputGate.getNumberOfInputChannels();

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve = new StatusWatermarkValve(
			numInputChannels,
			new ForwardingValveOutputHandler(streamOperator, lock));

	this.watermarkGauge = watermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);
}
 
Example #14
Source File: StreamTwoInputProcessor.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public StreamTwoInputProcessor(
		Collection<InputGate> inputGates1,
		Collection<InputGate> inputGates2,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		TwoInputStreamTask<IN1, IN2, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge input1WatermarkGauge,
		WatermarkGauge input2WatermarkGauge) throws IOException {

	final InputGate inputGate = InputGateUtil.createInputGate(inputGates1, inputGates2);

	this.barrierHandler = InputProcessorUtil.createCheckpointBarrierHandler(
		checkpointedTask, checkpointMode, ioManager, inputGate, taskManagerConfig);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN1> ser1 = new StreamElementSerializer<>(inputSerializer1);
	this.deserializationDelegate1 = new NonReusingDeserializationDelegate<>(ser1);

	StreamElementSerializer<IN2> ser2 = new StreamElementSerializer<>(inputSerializer2);
	this.deserializationDelegate2 = new NonReusingDeserializationDelegate<>(ser2);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	// determine which unioned channels belong to input 1 and which belong to input 2
	int numInputChannels1 = 0;
	for (InputGate gate: inputGates1) {
		numInputChannels1 += gate.getNumberOfInputChannels();
	}

	this.numInputChannels1 = numInputChannels1;
	this.numInputChannels2 = inputGate.getNumberOfInputChannels() - numInputChannels1;

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve1 = new StatusWatermarkValve(numInputChannels1, new ForwardingValveOutputHandler1(streamOperator, lock));
	this.statusWatermarkValve2 = new StatusWatermarkValve(numInputChannels2, new ForwardingValveOutputHandler2(streamOperator, lock));

	this.input1WatermarkGauge = input1WatermarkGauge;
	this.input2WatermarkGauge = input2WatermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);
}
 
Example #15
Source File: RecordWriter.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Sets the metric group for this RecordWriter.
    */
public void setMetricGroup(TaskIOMetricGroup metrics) {
	numBytesOut = metrics.getNumBytesOutCounter();
	numBuffersOut = metrics.getNumBuffersOutCounter();
}
 
Example #16
Source File: StreamOneInputProcessor.java    From flink with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public StreamOneInputProcessor(
		InputGate[] inputGates,
		TypeSerializer<IN> inputSerializer,
		StreamTask<?, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		OneInputStreamOperator<IN, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge watermarkGauge,
		String taskName,
		OperatorChain<?, ?> operatorChain) throws IOException {

	InputGate inputGate = InputGateUtil.createInputGate(inputGates);

	CheckpointedInputGate barrierHandler = InputProcessorUtil.createCheckpointedInputGate(
		checkpointedTask,
		checkpointMode,
		ioManager,
		inputGate,
		taskManagerConfig,
		taskName);
	this.input = new StreamTaskNetworkInput(barrierHandler, inputSerializer, ioManager, 0);

	this.lock = checkNotNull(lock);

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve = new StatusWatermarkValve(
		inputGate.getNumberOfInputChannels(),
		new ForwardingValveOutputHandler(streamOperator, lock));

	this.watermarkGauge = watermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);

	this.operatorChain = checkNotNull(operatorChain);
}
 
Example #17
Source File: StreamTwoInputProcessor.java    From flink with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public StreamTwoInputProcessor(
		Collection<InputGate> inputGates1,
		Collection<InputGate> inputGates2,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		TwoInputStreamTask<IN1, IN2, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator,
		TaskIOMetricGroup metrics,
		WatermarkGauge input1WatermarkGauge,
		WatermarkGauge input2WatermarkGauge,
		String taskName,
		OperatorChain<?, ?> operatorChain) throws IOException {

	final InputGate inputGate = InputGateUtil.createInputGate(inputGates1, inputGates2);

	this.barrierHandler = InputProcessorUtil.createCheckpointedInputGate(
		checkpointedTask,
		checkpointMode,
		ioManager,
		inputGate,
		taskManagerConfig,
		taskName);

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN1> ser1 = new StreamElementSerializer<>(inputSerializer1);
	this.deserializationDelegate1 = new NonReusingDeserializationDelegate<>(ser1);

	StreamElementSerializer<IN2> ser2 = new StreamElementSerializer<>(inputSerializer2);
	this.deserializationDelegate2 = new NonReusingDeserializationDelegate<>(ser2);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
			ioManager.getSpillingDirectoriesPaths());
	}

	// determine which unioned channels belong to input 1 and which belong to input 2
	int numInputChannels1 = 0;
	for (InputGate gate: inputGates1) {
		numInputChannels1 += gate.getNumberOfInputChannels();
	}

	this.numInputChannels1 = numInputChannels1;
	this.numInputChannels2 = inputGate.getNumberOfInputChannels() - numInputChannels1;

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve1 = new StatusWatermarkValve(numInputChannels1, new ForwardingValveOutputHandler1(streamOperator, lock));
	this.statusWatermarkValve2 = new StatusWatermarkValve(numInputChannels2, new ForwardingValveOutputHandler2(streamOperator, lock));

	this.input1WatermarkGauge = input1WatermarkGauge;
	this.input2WatermarkGauge = input2WatermarkGauge;
	metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos);

	this.operatorChain = checkNotNull(operatorChain);

	this.finishedChannels1 = new BitSet();
	this.finishedChannels2 = new BitSet();
}
 
Example #18
Source File: RecordWriter.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Sets the metric group for this RecordWriter.
    */
public void setMetricGroup(TaskIOMetricGroup metrics) {
	numBytesOut = metrics.getNumBytesOutCounter();
	numBuffersOut = metrics.getNumBuffersOutCounter();
	idleTimeMsPerSecond = metrics.getIdleTimeMsPerSecond();
}
 
Example #19
Source File: RecordWriter.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Sets the metric group for this RecordWriter.
    */
public void setMetricGroup(TaskIOMetricGroup metrics) {
	numBytesOut = metrics.getNumBytesOutCounter();
	numBuffersOut = metrics.getNumBuffersOutCounter();
}
 
Example #20
Source File: InputProcessorUtil.java    From flink with Apache License 2.0 4 votes vote down vote up
private static void registerCheckpointMetrics(TaskIOMetricGroup taskIOMetricGroup, CheckpointBarrierHandler barrierHandler) {
	taskIOMetricGroup.gauge(MetricNames.CHECKPOINT_ALIGNMENT_TIME, barrierHandler::getAlignmentDurationNanos);
	taskIOMetricGroup.gauge(MetricNames.CHECKPOINT_START_DELAY_TIME, barrierHandler::getCheckpointStartDelayNanos);
}