org.apache.flink.streaming.runtime.streamrecord.LatencyMarker Java Examples

The following examples show how to use org.apache.flink.streaming.runtime.streamrecord.LatencyMarker. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MultipleInputStreamTaskTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testLatencyMarker() throws Exception {
	final Map<String, Metric> metrics = new ConcurrentHashMap<>();
	final TaskMetricGroup taskMetricGroup = new StreamTaskTestHarness.TestTaskMetricGroup(metrics);

	try (StreamTaskMailboxTestHarness<String> testHarness =
			new MultipleInputStreamTaskTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO)
				.addInput(BasicTypeInfo.STRING_TYPE_INFO)
				.addInput(BasicTypeInfo.INT_TYPE_INFO)
				.addInput(BasicTypeInfo.DOUBLE_TYPE_INFO)
				.setupOutputForSingletonOperatorChain(new MapToStringMultipleInputOperatorFactory())
				.setTaskMetricGroup(taskMetricGroup)
				.build()) {
		ArrayDeque<Object> expectedOutput = new ArrayDeque<>();

		OperatorID sourceId = new OperatorID();
		LatencyMarker latencyMarker = new LatencyMarker(42L, sourceId, 0);
		testHarness.processElement(latencyMarker);
		expectedOutput.add(latencyMarker);

		assertThat(testHarness.getOutput(), contains(expectedOutput.toArray()));

		testHarness.endInput();
		testHarness.waitForTaskCompletion();
	}
}
Example #2
Source File: LatencyStatsTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static void testLatencyStats(
	final LatencyStats.Granularity granularity,
	final Consumer<List<Tuple2<String, Histogram>>> verifier) {

	final AbstractMetricGroup<?> dummyGroup = UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup();
	final TestMetricRegistry registry = new TestMetricRegistry();
	final MetricGroup parentGroup = new GenericMetricGroup(registry, dummyGroup, PARENT_GROUP_NAME);

	final LatencyStats latencyStats = new LatencyStats(
		parentGroup,
		MetricOptions.LATENCY_HISTORY_SIZE.defaultValue(),
		OPERATOR_SUBTASK_INDEX,
		OPERATOR_ID,
		granularity);

	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 0));
	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 0));
	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 1));
	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_2, 2));
	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_2, 3));

	verifier.accept(registry.latencyHistograms);
}
Example #3
Source File: LatencyStatsTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static void testLatencyStats(
	final LatencyStats.Granularity granularity,
	final Consumer<List<Tuple2<String, Histogram>>> verifier) {

	final AbstractMetricGroup<?> dummyGroup = UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup();
	final TestMetricRegistry registry = new TestMetricRegistry();
	final MetricGroup parentGroup = new GenericMetricGroup(registry, dummyGroup, PARENT_GROUP_NAME);

	final LatencyStats latencyStats = new LatencyStats(
		parentGroup,
		MetricOptions.LATENCY_HISTORY_SIZE.defaultValue(),
		OPERATOR_SUBTASK_INDEX,
		OPERATOR_ID,
		granularity);

	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 0));
	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 0));
	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_1, 1));
	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_2, 2));
	latencyStats.reportLatency(new LatencyMarker(0L, SOURCE_ID_2, 3));

	verifier.accept(registry.latencyHistograms);
}
Example #4
Source File: StreamSource.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public LatencyMarksEmitter(
		final ProcessingTimeService processingTimeService,
		final Output<StreamRecord<OUT>> output,
		long latencyTrackingInterval,
		final OperatorID operatorId,
		final int subtaskIndex) {

	latencyMarkTimer = processingTimeService.scheduleAtFixedRate(
		new ProcessingTimeCallback() {
			@Override
			public void onProcessingTime(long timestamp) throws Exception {
				try {
					// ProcessingTimeService callbacks are executed under the checkpointing lock
					output.emitLatencyMarker(new LatencyMarker(processingTimeService.getCurrentProcessingTime(), operatorId, subtaskIndex));
				} catch (Throwable t) {
					// we catch the Throwables here so that we don't trigger the processing
					// timer services async exception handler
					LOG.warn("Error while emitting latency marker.", t);
				}
			}
		},
		0L,
		latencyTrackingInterval);
}
Example #5
Source File: StreamSource.java    From flink with Apache License 2.0 6 votes vote down vote up 
public LatencyMarksEmitter(
		final ProcessingTimeService processingTimeService,
		final Output<StreamRecord<OUT>> output,
		long latencyTrackingInterval,
		final OperatorID operatorId,
		final int subtaskIndex) {

	latencyMarkTimer = processingTimeService.scheduleWithFixedDelay(
		new ProcessingTimeCallback() {
			@Override
			public void onProcessingTime(long timestamp) throws Exception {
				try {
					// ProcessingTimeService callbacks are executed under the checkpointing lock
					output.emitLatencyMarker(new LatencyMarker(processingTimeService.getCurrentProcessingTime(), operatorId, subtaskIndex));
				} catch (Throwable t) {
					// we catch the Throwables here so that we don't trigger the processing
					// timer services async exception handler
					LOG.warn("Error while emitting latency marker.", t);
				}
			}
		},
		0L,
		latencyTrackingInterval);
}
Example #6
Source File: RecordWriterOutput.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
	serializationDelegate.setInstance(latencyMarker);

	try {
		recordWriter.randomEmit(serializationDelegate);
	}
	catch (Exception e) {
		throw new RuntimeException(e.getMessage(), e);
	}
}
Example #7
Source File: SinkOperator.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected void reportOrForwardLatencyMarker(LatencyMarker marker) {
	// all operators are tracking latencies
	this.latencyStats.reportLatency(marker);

	// sinks don't forward latency markers
}
Example #8
Source File: OperatorChain.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
	if (outputs.length <= 0) {
		// ignore
	} else if (outputs.length == 1) {
		outputs[0].emitLatencyMarker(latencyMarker);
	} else {
		// randomly select an output
		outputs[random.nextInt(outputs.length)].emitLatencyMarker(latencyMarker);
	}
}
Example #9
Source File: AbstractStreamOperator.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
protected void reportOrForwardLatencyMarker(LatencyMarker marker) {
	// all operators are tracking latencies
	this.latencyStats.reportLatency(marker);

	// everything except sinks forwards latency markers
	this.output.emitLatencyMarker(marker);
}
Example #10
Source File: RecordWriterOutput.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
	serializationDelegate.setInstance(latencyMarker);

	try {
		recordWriter.randomEmit(serializationDelegate);
	}
	catch (Exception e) {
		throw new RuntimeException(e.getMessage(), e);
	}
}
Example #11
Source File: OperatorChain.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
	try {
		operator.processLatencyMarker(latencyMarker);
	}
	catch (Exception e) {
		throw new ExceptionInChainedOperatorException(e);
	}
}
Example #12
Source File: LatencyStats.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base
		.addGroup("source_id", String.valueOf(marker.getOperatorId()))
		.addGroup("source_subtask_index", String.valueOf(marker.getSubtaskIndex()));
}
Example #13
Source File: OperatorChain.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
	try {
		operator.processLatencyMarker(latencyMarker);
	}
	catch (Exception e) {
		throw new ExceptionInChainedOperatorException(e);
	}
}
Example #14
Source File: LatencyStats.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base
		.addGroup("source_id", String.valueOf(marker.getOperatorId()))
		.addGroup("source_subtask_index", String.valueOf(marker.getSubtaskIndex()));
}
Example #15
Source File: RecordWriterOutput.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
	serializationDelegate.setInstance(latencyMarker);

	try {
		recordWriter.randomEmit(serializationDelegate);
	}
	catch (Exception e) {
		throw new RuntimeException(e.getMessage(), e);
	}
}
Example #16
Source File: StreamSink.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected void reportOrForwardLatencyMarker(LatencyMarker marker) {
	// all operators are tracking latencies
	this.latencyStats.reportLatency(marker);

	// sinks don't forward latency markers
}
Example #17
Source File: StreamSink.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
protected void reportOrForwardLatencyMarker(LatencyMarker marker) {
	// all operators are tracking latencies
	this.latencyStats.reportLatency(marker);

	// sinks don't forward latency markers
}
Example #18
Source File: OperatorChain.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
	if (outputs.length <= 0) {
		// ignore
	} else if (outputs.length == 1) {
		outputs[0].emitLatencyMarker(latencyMarker);
	} else {
		// randomly select an output
		outputs[random.nextInt(outputs.length)].emitLatencyMarker(latencyMarker);
	}
}
Example #19
Source File: OperatorChain.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
	try {
		operator.processLatencyMarker(latencyMarker);
	}
	catch (Exception e) {
		throw new ExceptionInChainedOperatorException(e);
	}
}
Example #20
Source File: LatencyStats.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base
		.addGroup("source_id", String.valueOf(marker.getOperatorId()));
}
Example #21
Source File: LatencyStats.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base
		.addGroup("source_id", String.valueOf(marker.getOperatorId()))
		.addGroup("source_subtask_index", String.valueOf(marker.getSubtaskIndex()));
}
Example #22
Source File: LatencyStats.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base;
}
Example #23
Source File: LatencyStats.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base
		.addGroup("source_id", String.valueOf(marker.getOperatorId()));
}
Example #24
Source File: LatencyStats.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base;
}
Example #25
Source File: StreamTwoInputProcessor.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) throws Exception {
	if (inputIndex == 0) {
		operator.processLatencyMarker1(latencyMarker);
	} else {
		operator.processLatencyMarker2(latencyMarker);
	}
}
Example #26
Source File: LatencyStats.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base
		.addGroup("source_id", String.valueOf(marker.getOperatorId()));
}
Example #27
Source File: StreamSink.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected void reportOrForwardLatencyMarker(LatencyMarker marker) {
	// all operators are tracking latencies
	this.latencyStats.reportLatency(marker);

	// sinks don't forward latency markers
}
Example #28
Source File: AbstractStreamOperatorV2.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected void reportOrForwardLatencyMarker(LatencyMarker marker) {
	// all operators are tracking latencies
	this.latencyStats.reportLatency(marker);

	// everything except sinks forwards latency markers
	this.output.emitLatencyMarker(marker);
}
Example #29
Source File: StreamIterationTail.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public void emitLatencyMarker(LatencyMarker latencyMarker) {
}
Example #30
Source File: StreamGraphGeneratorTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public void processLatencyMarker2(LatencyMarker latencyMarker) throws Exception {
	// ignore
}