org.apache.flink.metrics.MetricGroup Java Examples

The following examples show how to use org.apache.flink.metrics.MetricGroup. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MetricGroupTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Verifies the basic behavior when defining user-defined variables.
 */
@Test
public void testUserDefinedVariable() {
	MetricRegistry registry = NoOpMetricRegistry.INSTANCE;
	GenericMetricGroup root = new GenericMetricGroup(registry, new DummyAbstractMetricGroup(registry), "root");

	String key = "key";
	String value = "value";
	MetricGroup group = root.addGroup(key, value);

	String variableValue = group.getAllVariables().get(ScopeFormat.asVariable("key"));
	assertEquals(value, variableValue);

	String identifier = group.getMetricIdentifier("metric");
	assertTrue("Key is missing from metric identifier.", identifier.contains("key"));
	assertTrue("Value is missing from metric identifier.", identifier.contains("value"));

	String logicalScope = ((AbstractMetricGroup) group).getLogicalScope(new DummyCharacterFilter());
	assertTrue("Key is missing from logical scope.", logicalScope.contains(key));
	assertFalse("Value is present in logical scope.", logicalScope.contains(value));
}
 
Example #2
Source File: NettyShuffleMetricFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
private static void registerInputMetrics(
		boolean isDetailedMetrics,
		MetricGroup inputGroup,
		MetricGroup buffersGroup,
		SingleInputGate[] inputGates) {
	if (isDetailedMetrics) {
		InputGateMetrics.registerQueueLengthMetrics(inputGroup, inputGates);
	}

	buffersGroup.gauge(METRIC_INPUT_QUEUE_LENGTH, new InputBuffersGauge(inputGates));

	FloatingBuffersUsageGauge floatingBuffersUsageGauge = new FloatingBuffersUsageGauge(inputGates);
	ExclusiveBuffersUsageGauge exclusiveBuffersUsageGauge = new ExclusiveBuffersUsageGauge(inputGates);
	CreditBasedInputBuffersUsageGauge creditBasedInputBuffersUsageGauge = new CreditBasedInputBuffersUsageGauge(
		floatingBuffersUsageGauge,
		exclusiveBuffersUsageGauge,
		inputGates);
	buffersGroup.gauge(METRIC_INPUT_EXCLUSIVE_BUFFERS_USAGE, exclusiveBuffersUsageGauge);
	buffersGroup.gauge(METRIC_INPUT_FLOATING_BUFFERS_USAGE, floatingBuffersUsageGauge);
	buffersGroup.gauge(METRIC_INPUT_POOL_USAGE, creditBasedInputBuffersUsageGauge);
}
 
Example #3
Source File: MetricGroupTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Verifies the basic behavior when defining user-defined variables.
 */
@Test
public void testUserDefinedVariable() {
	MetricRegistry registry = NoOpMetricRegistry.INSTANCE;
	GenericMetricGroup root = new GenericMetricGroup(registry, new DummyAbstractMetricGroup(registry), "root");

	String key = "key";
	String value = "value";
	MetricGroup group = root.addGroup(key, value);

	String variableValue = group.getAllVariables().get(ScopeFormat.asVariable("key"));
	assertEquals(value, variableValue);

	String identifier = group.getMetricIdentifier("metric");
	assertTrue("Key is missing from metric identifier.", identifier.contains("key"));
	assertTrue("Value is missing from metric identifier.", identifier.contains("value"));

	String logicalScope = ((AbstractMetricGroup) group).getLogicalScope(new DummyCharacterFilter());
	assertTrue("Key is missing from logical scope.", logicalScope.contains(key));
	assertFalse("Value is present in logical scope.", logicalScope.contains(value));
}
 
Example #4
Source File: MockStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) {
	return new MockKeyedStateBackendBuilder<>(
		new KvStateRegistry().createTaskRegistry(jobID, new JobVertexID()),
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		cancelStreamRegistry).build();
}
 
Example #5
Source File: DatadogHttpReporter.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void notifyOfAddedMetric(Metric metric, String metricName, MetricGroup group) {
	final String name = group.getMetricIdentifier(metricName);

	List<String> tags = new ArrayList<>(configTags);
	tags.addAll(getTagsFromMetricGroup(group));
	String host = getHostFromMetricGroup(group);

	if (metric instanceof Counter) {
		Counter c = (Counter) metric;
		counters.put(c, new DCounter(c, name, host, tags));
	} else if (metric instanceof Gauge) {
		Gauge g = (Gauge) metric;
		gauges.put(g, new DGauge(g, name, host, tags));
	} else if (metric instanceof Meter) {
		Meter m = (Meter) metric;
		// Only consider rate
		meters.put(m, new DMeter(m, name, host, tags));
	} else if (metric instanceof Histogram) {
		LOGGER.warn("Cannot add {} because Datadog HTTP API doesn't support Histogram", metricName);
	} else {
		LOGGER.warn("Cannot add unknown metric type {}. This indicates that the reporter " +
			"does not support this metric type.", metric.getClass().getName());
	}
}
 
Example #6
Source File: JMXReporter.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void notifyOfRemovedMetric(Metric metric, String metricName, MetricGroup group) {
	try {
		synchronized (this) {
			final ObjectName jmxName = registeredMetrics.remove(metric);

			// remove the metric if it is known. if it is not known, ignore the request
			if (jmxName != null) {
				mBeanServer.unregisterMBean(jmxName);
			}
		}
	} catch (InstanceNotFoundException e) {
		// alright then
	} catch (Throwable t) {
		// never propagate exceptions - the metrics reporter should not affect the stability
		// of the running system
		LOG.error("Un-registering metric failed", t);
	}
}
 
Example #7
Source File: AbstractReporter.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public void notifyOfAddedMetric(Metric metric, String metricName, MetricGroup group) {
	final String name = group.getMetricIdentifier(metricName, this);

	synchronized (this) {
		if (metric instanceof Counter) {
			counters.put((Counter) metric, name);
		} else if (metric instanceof Gauge) {
			gauges.put((Gauge<?>) metric, name);
		} else if (metric instanceof Histogram) {
			histograms.put((Histogram) metric, name);
		} else if (metric instanceof Meter) {
			meters.put((Meter) metric, name);
		} else {
			log.warn("Cannot add unknown metric type {}. This indicates that the reporter " +
				"does not support this metric type.", metric.getClass().getName());
		}
	}
}
 
Example #8
Source File: MetricGroupTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that existing key/value groups are returned when calling {@link MetricGroup#addGroup(String)}.
 */
@Test
public void testNameCollisionAfterKeyValueGroup() {
	MetricRegistry registry = NoOpMetricRegistry.INSTANCE;
	GenericMetricGroup root = new GenericMetricGroup(registry, new DummyAbstractMetricGroup(registry), "root");

	String key = "key";
	String value = "value";

	root.addGroup(key, value);
	MetricGroup group = root.addGroup(key).addGroup(value);

	String variableValue = group.getAllVariables().get(ScopeFormat.asVariable("key"));
	assertEquals(value, variableValue);

	String identifier = group.getMetricIdentifier("metric");
	assertTrue("Key is missing from metric identifier.", identifier.contains("key"));
	assertTrue("Value is missing from metric identifier.", identifier.contains("value"));

	String logicalScope = ((AbstractMetricGroup) group).getLogicalScope(new DummyCharacterFilter());
	assertTrue("Key is missing from logical scope.", logicalScope.contains(key));
	assertFalse("Value is present in logical scope.", logicalScope.contains(value));
}
 
Example #9
Source File: SystemResourcesMetricsInitializer.java    From flink with Apache License 2.0 6 votes vote down vote up
private static void instantiateCPUMetrics(MetricGroup metrics, SystemResourcesCounter usageCounter) {
	metrics.<Double, Gauge<Double>>gauge("Usage", usageCounter::getCpuUsage);
	metrics.<Double, Gauge<Double>>gauge("Idle", usageCounter::getCpuIdle);
	metrics.<Double, Gauge<Double>>gauge("Sys", usageCounter::getCpuSys);
	metrics.<Double, Gauge<Double>>gauge("User", usageCounter::getCpuUser);
	metrics.<Double, Gauge<Double>>gauge("IOWait", usageCounter::getIOWait);
	metrics.<Double, Gauge<Double>>gauge("Nice", usageCounter::getCpuNice);
	metrics.<Double, Gauge<Double>>gauge("Irq", usageCounter::getCpuIrq);
	metrics.<Double, Gauge<Double>>gauge("SoftIrq", usageCounter::getCpuSoftIrq);

	metrics.<Double, Gauge<Double>>gauge("Load1min", usageCounter::getCpuLoad1);
	metrics.<Double, Gauge<Double>>gauge("Load5min", usageCounter::getCpuLoad5);
	metrics.<Double, Gauge<Double>>gauge("Load15min", usageCounter::getCpuLoad15);

	for (int i = 0; i < usageCounter.getProcessorsCount(); i++) {
		final int processor = i;
		metrics.<Double, Gauge<Double>>gauge(
			String.format("UsageCPU%d", processor),
			() -> usageCounter.getCpuUsagePerProcessor(processor));
	}
}
 
Example #10
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 6 votes vote down vote up
protected TestingFetcher(
		SourceFunction.SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> seedPartitionsWithInitialOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		ProcessingTimeService processingTimeProvider,
		long autoWatermarkInterval,
		ClassLoader userCodeClassLoader,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	super(
			sourceContext,
			seedPartitionsWithInitialOffsets,
			watermarkStrategy,
			processingTimeProvider,
			autoWatermarkInterval,
			userCodeClassLoader,
			consumerMetricGroup,
			useMetrics);
}
 
Example #11
Source File: MetricUtils.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public static JobManagerMetricGroup instantiateJobManagerMetricGroup(
		final MetricRegistry metricRegistry,
		final String hostname,
		final Optional<Time> systemResourceProbeInterval) {
	final JobManagerMetricGroup jobManagerMetricGroup = new JobManagerMetricGroup(
		metricRegistry,
		hostname);

	MetricGroup statusGroup = jobManagerMetricGroup.addGroup(METRIC_GROUP_STATUS_NAME);

	// initialize the JM metrics
	instantiateStatusMetrics(statusGroup);

	if (systemResourceProbeInterval.isPresent()) {
		instantiateSystemMetrics(jobManagerMetricGroup, systemResourceProbeInterval.get());
	}
	return jobManagerMetricGroup;
}
 
Example #12
Source File: MetricGroupRegistrationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that when attempting to create a group with the name of an existing one the existing one will be returned instead.
 */
@Test
public void testDuplicateGroupName() throws Exception {
	Configuration config = new Configuration();

	MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(config));

	MetricGroup root = new TaskManagerMetricGroup(registry, "host", "id");

	MetricGroup group1 = root.addGroup("group");
	MetricGroup group2 = root.addGroup("group");
	MetricGroup group3 = root.addGroup("group");
	Assert.assertTrue(group1 == group2 && group2 == group3);

	registry.shutdown().get();
}
 
Example #13
Source File: KafkaConsumerThread.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public KafkaConsumerThread(
		Logger log,
		Handover handover,
		Properties kafkaProperties,
		ClosableBlockingQueue<KafkaTopicPartitionState<TopicPartition>> unassignedPartitionsQueue,
		KafkaConsumerCallBridge09 consumerCallBridge,
		String threadName,
		long pollTimeout,
		boolean useMetrics,
		MetricGroup consumerMetricGroup,
		MetricGroup subtaskMetricGroup,
		FlinkConnectorRateLimiter rateLimiter) {

	super(threadName);
	setDaemon(true);

	this.log = checkNotNull(log);
	this.handover = checkNotNull(handover);
	this.kafkaProperties = checkNotNull(kafkaProperties);
	this.consumerMetricGroup = checkNotNull(consumerMetricGroup);
	this.subtaskMetricGroup = checkNotNull(subtaskMetricGroup);
	this.consumerCallBridge = checkNotNull(consumerCallBridge);

	this.unassignedPartitionsQueue = checkNotNull(unassignedPartitionsQueue);

	this.pollTimeout = pollTimeout;
	this.useMetrics = useMetrics;

	this.consumerReassignmentLock = new Object();
	this.nextOffsetsToCommit = new AtomicReference<>();
	this.running = true;

	if (rateLimiter != null) {
		this.rateLimiter = rateLimiter;
	}
}
 
Example #14
Source File: CollectionExecutor.java    From flink with Apache License 2.0 5 votes vote down vote up
private <IN1, IN2, OUT> List<OUT> executeBinaryOperator(DualInputOperator<?, ?, ?, ?> operator, int superStep) throws Exception {
	Operator<?> inputOp1 = operator.getFirstInput();
	Operator<?> inputOp2 = operator.getSecondInput();
	
	if (inputOp1 == null) {
		throw new InvalidProgramException("The binary operation " + operator.getName() + " has no first input.");
	}
	if (inputOp2 == null) {
		throw new InvalidProgramException("The binary operation " + operator.getName() + " has no second input.");
	}
	
	// compute inputs
	@SuppressWarnings("unchecked")
	List<IN1> inputData1 = (List<IN1>) execute(inputOp1, superStep);
	@SuppressWarnings("unchecked")
	List<IN2> inputData2 = (List<IN2>) execute(inputOp2, superStep);
	
	@SuppressWarnings("unchecked")
	DualInputOperator<IN1, IN2, OUT, ?> typedOp = (DualInputOperator<IN1, IN2, OUT, ?>) operator;
	
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();

	if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
			new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
		
		for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
			List<?> bcData = execute(bcInputs.getValue());
			ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
		}
	} else {
		ctx = null;
	}

	return typedOp.executeOnCollections(inputData1, inputData2, ctx, executionConfig);
}
 
Example #15
Source File: FlinkKafkaShuffleConsumer.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	return new KafkaShuffleFetcher<>(
		sourceContext,
		assignedPartitionsWithInitialOffsets,
		watermarkStrategy,
		runtimeContext.getProcessingTimeService(),
		runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
		runtimeContext.getUserCodeClassLoader(),
		runtimeContext.getTaskNameWithSubtasks(),
		deserializer,
		properties,
		pollTimeout,
		runtimeContext.getMetricGroup(),
		consumerMetricGroup,
		useMetrics,
		typeSerializer,
		producerParallelism);
}
 
Example #16
Source File: CheckpointStatsTracker.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Register the exposed metrics.
 *
 * @param metricGroup Metric group to use for the metrics.
 */
private void registerMetrics(MetricGroup metricGroup) {
	metricGroup.gauge(NUMBER_OF_CHECKPOINTS_METRIC, new CheckpointsCounter());
	metricGroup.gauge(NUMBER_OF_IN_PROGRESS_CHECKPOINTS_METRIC, new InProgressCheckpointsCounter());
	metricGroup.gauge(NUMBER_OF_COMPLETED_CHECKPOINTS_METRIC, new CompletedCheckpointsCounter());
	metricGroup.gauge(NUMBER_OF_FAILED_CHECKPOINTS_METRIC, new FailedCheckpointsCounter());
	metricGroup.gauge(LATEST_RESTORED_CHECKPOINT_TIMESTAMP_METRIC, new LatestRestoredCheckpointTimestampGauge());
	metricGroup.gauge(LATEST_COMPLETED_CHECKPOINT_SIZE_METRIC, new LatestCompletedCheckpointSizeGauge());
	metricGroup.gauge(LATEST_COMPLETED_CHECKPOINT_DURATION_METRIC, new LatestCompletedCheckpointDurationGauge());
	metricGroup.gauge(LATEST_COMPLETED_CHECKPOINT_EXTERNAL_PATH_METRIC, new LatestCompletedCheckpointExternalPathGauge());
}
 
Example #17
Source File: FlinkKafkaConsumer09.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {

	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	// If a rateLimiter is set, then call rateLimiter.open() with the runtime context.
	if (rateLimiter != null) {
		rateLimiter.open(runtimeContext);
	}

	return new Kafka09Fetcher<>(
			sourceContext,
			assignedPartitionsWithInitialOffsets,
			watermarksPeriodic,
			watermarksPunctuated,
			runtimeContext.getProcessingTimeService(),
			runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
			runtimeContext.getUserCodeClassLoader(),
			runtimeContext.getTaskNameWithSubtasks(),
			deserializer,
			properties,
			pollTimeout,
			runtimeContext.getMetricGroup(),
			consumerMetricGroup,
			useMetrics,
			rateLimiter);
}
 
Example #18
Source File: AbstractStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public abstract <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws IOException;
 
Example #19
Source File: FlinkKafkaConsumerBaseMigrationTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> thisSubtaskPartitionsWithStartOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	return fetcher;
}
 
Example #20
Source File: AbstractPrometheusReporter.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void notifyOfAddedMetric(final Metric metric, final String metricName, final MetricGroup group) {

	List<String> dimensionKeys = new LinkedList<>();
	List<String> dimensionValues = new LinkedList<>();
	for (final Map.Entry<String, String> dimension : group.getAllVariables().entrySet()) {
		final String key = dimension.getKey();
		dimensionKeys.add(CHARACTER_FILTER.filterCharacters(key.substring(1, key.length() - 1)));
		dimensionValues.add(labelValueCharactersFilter.filterCharacters(dimension.getValue()));
	}

	final String scopedMetricName = getScopedName(metricName, group);
	final String helpString = metricName + " (scope: " + getLogicalScope(group) + ")";

	final Collector collector;
	Integer count = 0;

	synchronized (this) {
		if (collectorsWithCountByMetricName.containsKey(scopedMetricName)) {
			final AbstractMap.SimpleImmutableEntry<Collector, Integer> collectorWithCount = collectorsWithCountByMetricName.get(scopedMetricName);
			collector = collectorWithCount.getKey();
			count = collectorWithCount.getValue();
		} else {
			collector = createCollector(metric, dimensionKeys, dimensionValues, scopedMetricName, helpString);
			try {
				collector.register();
			} catch (Exception e) {
				log.warn("There was a problem registering metric {}.", metricName, e);
			}
		}
		addMetric(metric, dimensionValues, collector);
		collectorsWithCountByMetricName.put(scopedMetricName, new AbstractMap.SimpleImmutableEntry<>(collector, count + 1));
	}
}
 
Example #21
Source File: CollectionExecutor.java    From flink with Apache License 2.0 5 votes vote down vote up
private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception {
	Operator<?> inputOp = sink.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The data sink " + sink.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> input = (List<IN>) execute(inputOp);
	
	@SuppressWarnings("unchecked")
	GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink;

	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
		
	if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}

	typedSink.executeOnCollections(input, ctx, executionConfig);
}
 
Example #22
Source File: NettyShuffleMetricFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
public static void registerShuffleMetrics(
		MetricGroup metricGroup,
		NetworkBufferPool networkBufferPool) {
	checkNotNull(metricGroup);
	checkNotNull(networkBufferPool);

	//noinspection deprecation
	registerShuffleMetrics(METRIC_GROUP_NETWORK_DEPRECATED, metricGroup, networkBufferPool);
	registerShuffleMetrics(METRIC_GROUP_NETTY, metricGroup.addGroup(METRIC_GROUP_SHUFFLE), networkBufferPool);
}
 
Example #23
Source File: RocksDBNativeMetricMonitor.java    From flink with Apache License 2.0 5 votes vote down vote up
public RocksDBNativeMetricMonitor(
	@Nonnull RocksDBNativeMetricOptions options,
	@Nonnull MetricGroup metricGroup,
	@Nonnull RocksDB rocksDB
) {
	this.options = options;
	this.metricGroup = metricGroup;
	this.rocksDB = rocksDB;

	this.lock = new Object();
}
 
Example #24
Source File: LatencyStats.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base;
}
 
Example #25
Source File: StateBackendITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws IOException {
	throw new SuccessException();
}
 
Example #26
Source File: CollectionExecutor.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private <IN1, IN2, OUT> List<OUT> executeBinaryOperator(DualInputOperator<?, ?, ?, ?> operator, int superStep) throws Exception {
	Operator<?> inputOp1 = operator.getFirstInput();
	Operator<?> inputOp2 = operator.getSecondInput();
	
	if (inputOp1 == null) {
		throw new InvalidProgramException("The binary operation " + operator.getName() + " has no first input.");
	}
	if (inputOp2 == null) {
		throw new InvalidProgramException("The binary operation " + operator.getName() + " has no second input.");
	}
	
	// compute inputs
	@SuppressWarnings("unchecked")
	List<IN1> inputData1 = (List<IN1>) execute(inputOp1, superStep);
	@SuppressWarnings("unchecked")
	List<IN2> inputData2 = (List<IN2>) execute(inputOp2, superStep);
	
	@SuppressWarnings("unchecked")
	DualInputOperator<IN1, IN2, OUT, ?> typedOp = (DualInputOperator<IN1, IN2, OUT, ?>) operator;
	
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();

	if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
			new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
		
		for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
			List<?> bcData = execute(bcInputs.getValue());
			ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
		}
	} else {
		ctx = null;
	}

	return typedOp.executeOnCollections(inputData1, inputData2, ctx, executionConfig);
}
 
Example #27
Source File: RocksDBNativeMetricMonitor.java    From flink with Apache License 2.0 5 votes vote down vote up
public RocksDBNativeMetricMonitor(
	@Nonnull RocksDBNativeMetricOptions options,
	@Nonnull MetricGroup metricGroup,
	@Nonnull RocksDB rocksDB
) {
	this.options = options;
	this.metricGroup = metricGroup;
	this.rocksDB = rocksDB;

	this.lock = new Object();
}
 
Example #28
Source File: MetricRegistryImplTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void notifyOfAddedMetric(Metric metric, String metricName, MetricGroup group) {
	String expectedMetric = "A" + expectedDelimiter + "B" + expectedDelimiter + "C";
	assertEquals(expectedMetric, group.getMetricIdentifier(metricName, this));
	assertEquals(expectedMetric, group.getMetricIdentifier(metricName));
	numCorrectDelimitersForRegister++;
}
 
Example #29
Source File: AbstractRuntimeUDFContext.java    From flink with Apache License 2.0 5 votes vote down vote up
public AbstractRuntimeUDFContext(TaskInfo taskInfo,
									ClassLoader userCodeClassLoader,
									ExecutionConfig executionConfig,
									Map<String, Accumulator<?, ?>> accumulators,
									Map<String, Future<Path>> cpTasks,
									MetricGroup metrics) {
	this.taskInfo = checkNotNull(taskInfo);
	this.userCodeClassLoader = userCodeClassLoader;
	this.executionConfig = executionConfig;
	this.distributedCache = new DistributedCache(checkNotNull(cpTasks));
	this.accumulators = checkNotNull(accumulators);
	this.metrics = metrics;
}
 
Example #30
Source File: LatencyStats.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base;
}