Java Code Examples for org.apache.flink.metrics.MetricGroup

The following examples show how to use org.apache.flink.metrics.MetricGroup. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: MetricGroupTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verifies the basic behavior when defining user-defined variables.
 */
@Test
public void testUserDefinedVariable() {
	MetricRegistry registry = NoOpMetricRegistry.INSTANCE;
	GenericMetricGroup root = new GenericMetricGroup(registry, new DummyAbstractMetricGroup(registry), "root");

	String key = "key";
	String value = "value";
	MetricGroup group = root.addGroup(key, value);

	String variableValue = group.getAllVariables().get(ScopeFormat.asVariable("key"));
	assertEquals(value, variableValue);

	String identifier = group.getMetricIdentifier("metric");
	assertTrue("Key is missing from metric identifier.", identifier.contains("key"));
	assertTrue("Value is missing from metric identifier.", identifier.contains("value"));

	String logicalScope = ((AbstractMetricGroup) group).getLogicalScope(new DummyCharacterFilter());
	assertTrue("Key is missing from logical scope.", logicalScope.contains(key));
	assertFalse("Value is present in logical scope.", logicalScope.contains(value));
}
 
Example 2
Source Project: flink   Source File: NettyShuffleMetricFactory.java    License: Apache License 2.0 6 votes vote down vote up
private static void registerInputMetrics(
		boolean isDetailedMetrics,
		MetricGroup inputGroup,
		MetricGroup buffersGroup,
		SingleInputGate[] inputGates) {
	if (isDetailedMetrics) {
		InputGateMetrics.registerQueueLengthMetrics(inputGroup, inputGates);
	}

	buffersGroup.gauge(METRIC_INPUT_QUEUE_LENGTH, new InputBuffersGauge(inputGates));

	FloatingBuffersUsageGauge floatingBuffersUsageGauge = new FloatingBuffersUsageGauge(inputGates);
	ExclusiveBuffersUsageGauge exclusiveBuffersUsageGauge = new ExclusiveBuffersUsageGauge(inputGates);
	CreditBasedInputBuffersUsageGauge creditBasedInputBuffersUsageGauge = new CreditBasedInputBuffersUsageGauge(
		floatingBuffersUsageGauge,
		exclusiveBuffersUsageGauge,
		inputGates);
	buffersGroup.gauge(METRIC_INPUT_EXCLUSIVE_BUFFERS_USAGE, exclusiveBuffersUsageGauge);
	buffersGroup.gauge(METRIC_INPUT_FLOATING_BUFFERS_USAGE, floatingBuffersUsageGauge);
	buffersGroup.gauge(METRIC_INPUT_POOL_USAGE, creditBasedInputBuffersUsageGauge);
}
 
Example 3
Source Project: flink   Source File: MetricGroupTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verifies the basic behavior when defining user-defined variables.
 */
@Test
public void testUserDefinedVariable() {
	MetricRegistry registry = NoOpMetricRegistry.INSTANCE;
	GenericMetricGroup root = new GenericMetricGroup(registry, new DummyAbstractMetricGroup(registry), "root");

	String key = "key";
	String value = "value";
	MetricGroup group = root.addGroup(key, value);

	String variableValue = group.getAllVariables().get(ScopeFormat.asVariable("key"));
	assertEquals(value, variableValue);

	String identifier = group.getMetricIdentifier("metric");
	assertTrue("Key is missing from metric identifier.", identifier.contains("key"));
	assertTrue("Value is missing from metric identifier.", identifier.contains("value"));

	String logicalScope = ((AbstractMetricGroup) group).getLogicalScope(new DummyCharacterFilter());
	assertTrue("Key is missing from logical scope.", logicalScope.contains(key));
	assertFalse("Value is present in logical scope.", logicalScope.contains(value));
}
 
Example 4
Source Project: flink   Source File: MockStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) {
	return new MockKeyedStateBackendBuilder<>(
		new KvStateRegistry().createTaskRegistry(jobID, new JobVertexID()),
		keySerializer,
		env.getUserClassLoader(),
		numberOfKeyGroups,
		keyGroupRange,
		env.getExecutionConfig(),
		ttlTimeProvider,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()),
		cancelStreamRegistry).build();
}
 
Example 5
Source Project: flink   Source File: DatadogHttpReporter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void notifyOfAddedMetric(Metric metric, String metricName, MetricGroup group) {
	final String name = group.getMetricIdentifier(metricName);

	List<String> tags = new ArrayList<>(configTags);
	tags.addAll(getTagsFromMetricGroup(group));
	String host = getHostFromMetricGroup(group);

	if (metric instanceof Counter) {
		Counter c = (Counter) metric;
		counters.put(c, new DCounter(c, name, host, tags));
	} else if (metric instanceof Gauge) {
		Gauge g = (Gauge) metric;
		gauges.put(g, new DGauge(g, name, host, tags));
	} else if (metric instanceof Meter) {
		Meter m = (Meter) metric;
		// Only consider rate
		meters.put(m, new DMeter(m, name, host, tags));
	} else if (metric instanceof Histogram) {
		LOGGER.warn("Cannot add {} because Datadog HTTP API doesn't support Histogram", metricName);
	} else {
		LOGGER.warn("Cannot add unknown metric type {}. This indicates that the reporter " +
			"does not support this metric type.", metric.getClass().getName());
	}
}
 
Example 6
Source Project: flink   Source File: JMXReporter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void notifyOfRemovedMetric(Metric metric, String metricName, MetricGroup group) {
	try {
		synchronized (this) {
			final ObjectName jmxName = registeredMetrics.remove(metric);

			// remove the metric if it is known. if it is not known, ignore the request
			if (jmxName != null) {
				mBeanServer.unregisterMBean(jmxName);
			}
		}
	} catch (InstanceNotFoundException e) {
		// alright then
	} catch (Throwable t) {
		// never propagate exceptions - the metrics reporter should not affect the stability
		// of the running system
		LOG.error("Un-registering metric failed", t);
	}
}
 
Example 7
Source Project: Flink-CEPplus   Source File: AbstractReporter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void notifyOfAddedMetric(Metric metric, String metricName, MetricGroup group) {
	final String name = group.getMetricIdentifier(metricName, this);

	synchronized (this) {
		if (metric instanceof Counter) {
			counters.put((Counter) metric, name);
		} else if (metric instanceof Gauge) {
			gauges.put((Gauge<?>) metric, name);
		} else if (metric instanceof Histogram) {
			histograms.put((Histogram) metric, name);
		} else if (metric instanceof Meter) {
			meters.put((Meter) metric, name);
		} else {
			log.warn("Cannot add unknown metric type {}. This indicates that the reporter " +
				"does not support this metric type.", metric.getClass().getName());
		}
	}
}
 
Example 8
Source Project: flink   Source File: MetricGroupTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that existing key/value groups are returned when calling {@link MetricGroup#addGroup(String)}.
 */
@Test
public void testNameCollisionAfterKeyValueGroup() {
	MetricRegistry registry = NoOpMetricRegistry.INSTANCE;
	GenericMetricGroup root = new GenericMetricGroup(registry, new DummyAbstractMetricGroup(registry), "root");

	String key = "key";
	String value = "value";

	root.addGroup(key, value);
	MetricGroup group = root.addGroup(key).addGroup(value);

	String variableValue = group.getAllVariables().get(ScopeFormat.asVariable("key"));
	assertEquals(value, variableValue);

	String identifier = group.getMetricIdentifier("metric");
	assertTrue("Key is missing from metric identifier.", identifier.contains("key"));
	assertTrue("Value is missing from metric identifier.", identifier.contains("value"));

	String logicalScope = ((AbstractMetricGroup) group).getLogicalScope(new DummyCharacterFilter());
	assertTrue("Key is missing from logical scope.", logicalScope.contains(key));
	assertFalse("Value is present in logical scope.", logicalScope.contains(value));
}
 
Example 9
Source Project: flink   Source File: SystemResourcesMetricsInitializer.java    License: Apache License 2.0 6 votes vote down vote up
private static void instantiateCPUMetrics(MetricGroup metrics, SystemResourcesCounter usageCounter) {
	metrics.<Double, Gauge<Double>>gauge("Usage", usageCounter::getCpuUsage);
	metrics.<Double, Gauge<Double>>gauge("Idle", usageCounter::getCpuIdle);
	metrics.<Double, Gauge<Double>>gauge("Sys", usageCounter::getCpuSys);
	metrics.<Double, Gauge<Double>>gauge("User", usageCounter::getCpuUser);
	metrics.<Double, Gauge<Double>>gauge("IOWait", usageCounter::getIOWait);
	metrics.<Double, Gauge<Double>>gauge("Nice", usageCounter::getCpuNice);
	metrics.<Double, Gauge<Double>>gauge("Irq", usageCounter::getCpuIrq);
	metrics.<Double, Gauge<Double>>gauge("SoftIrq", usageCounter::getCpuSoftIrq);

	metrics.<Double, Gauge<Double>>gauge("Load1min", usageCounter::getCpuLoad1);
	metrics.<Double, Gauge<Double>>gauge("Load5min", usageCounter::getCpuLoad5);
	metrics.<Double, Gauge<Double>>gauge("Load15min", usageCounter::getCpuLoad15);

	for (int i = 0; i < usageCounter.getProcessorsCount(); i++) {
		final int processor = i;
		metrics.<Double, Gauge<Double>>gauge(
			String.format("UsageCPU%d", processor),
			() -> usageCounter.getCpuUsagePerProcessor(processor));
	}
}
 
Example 10
Source Project: flink   Source File: FlinkKafkaConsumerBaseTest.java    License: Apache License 2.0 6 votes vote down vote up
protected TestingFetcher(
		SourceFunction.SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> seedPartitionsWithInitialOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		ProcessingTimeService processingTimeProvider,
		long autoWatermarkInterval,
		ClassLoader userCodeClassLoader,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	super(
			sourceContext,
			seedPartitionsWithInitialOffsets,
			watermarkStrategy,
			processingTimeProvider,
			autoWatermarkInterval,
			userCodeClassLoader,
			consumerMetricGroup,
			useMetrics);
}
 
Example 11
Source Project: Flink-CEPplus   Source File: MetricUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static JobManagerMetricGroup instantiateJobManagerMetricGroup(
		final MetricRegistry metricRegistry,
		final String hostname,
		final Optional<Time> systemResourceProbeInterval) {
	final JobManagerMetricGroup jobManagerMetricGroup = new JobManagerMetricGroup(
		metricRegistry,
		hostname);

	MetricGroup statusGroup = jobManagerMetricGroup.addGroup(METRIC_GROUP_STATUS_NAME);

	// initialize the JM metrics
	instantiateStatusMetrics(statusGroup);

	if (systemResourceProbeInterval.isPresent()) {
		instantiateSystemMetrics(jobManagerMetricGroup, systemResourceProbeInterval.get());
	}
	return jobManagerMetricGroup;
}
 
Example 12
Source Project: flink   Source File: MetricGroupRegistrationTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verifies that when attempting to create a group with the name of an existing one the existing one will be returned instead.
 */
@Test
public void testDuplicateGroupName() throws Exception {
	Configuration config = new Configuration();

	MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(config));

	MetricGroup root = new TaskManagerMetricGroup(registry, "host", "id");

	MetricGroup group1 = root.addGroup("group");
	MetricGroup group2 = root.addGroup("group");
	MetricGroup group3 = root.addGroup("group");
	Assert.assertTrue(group1 == group2 && group2 == group3);

	registry.shutdown().get();
}
 
Example 13
Source Project: Flink-CEPplus   Source File: KafkaConsumerThread.java    License: Apache License 2.0 5 votes vote down vote up
public KafkaConsumerThread(
		Logger log,
		Handover handover,
		Properties kafkaProperties,
		ClosableBlockingQueue<KafkaTopicPartitionState<TopicPartition>> unassignedPartitionsQueue,
		KafkaConsumerCallBridge09 consumerCallBridge,
		String threadName,
		long pollTimeout,
		boolean useMetrics,
		MetricGroup consumerMetricGroup,
		MetricGroup subtaskMetricGroup,
		FlinkConnectorRateLimiter rateLimiter) {

	super(threadName);
	setDaemon(true);

	this.log = checkNotNull(log);
	this.handover = checkNotNull(handover);
	this.kafkaProperties = checkNotNull(kafkaProperties);
	this.consumerMetricGroup = checkNotNull(consumerMetricGroup);
	this.subtaskMetricGroup = checkNotNull(subtaskMetricGroup);
	this.consumerCallBridge = checkNotNull(consumerCallBridge);

	this.unassignedPartitionsQueue = checkNotNull(unassignedPartitionsQueue);

	this.pollTimeout = pollTimeout;
	this.useMetrics = useMetrics;

	this.consumerReassignmentLock = new Object();
	this.nextOffsetsToCommit = new AtomicReference<>();
	this.running = true;

	if (rateLimiter != null) {
		this.rateLimiter = rateLimiter;
	}
}
 
Example 14
Source Project: flink   Source File: CollectionExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private <IN1, IN2, OUT> List<OUT> executeBinaryOperator(DualInputOperator<?, ?, ?, ?> operator, int superStep) throws Exception {
	Operator<?> inputOp1 = operator.getFirstInput();
	Operator<?> inputOp2 = operator.getSecondInput();
	
	if (inputOp1 == null) {
		throw new InvalidProgramException("The binary operation " + operator.getName() + " has no first input.");
	}
	if (inputOp2 == null) {
		throw new InvalidProgramException("The binary operation " + operator.getName() + " has no second input.");
	}
	
	// compute inputs
	@SuppressWarnings("unchecked")
	List<IN1> inputData1 = (List<IN1>) execute(inputOp1, superStep);
	@SuppressWarnings("unchecked")
	List<IN2> inputData2 = (List<IN2>) execute(inputOp2, superStep);
	
	@SuppressWarnings("unchecked")
	DualInputOperator<IN1, IN2, OUT, ?> typedOp = (DualInputOperator<IN1, IN2, OUT, ?>) operator;
	
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();

	if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
			new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
		
		for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
			List<?> bcData = execute(bcInputs.getValue());
			ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
		}
	} else {
		ctx = null;
	}

	return typedOp.executeOnCollections(inputData1, inputData2, ctx, executionConfig);
}
 
Example 15
Source Project: flink   Source File: FlinkKafkaShuffleConsumer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	return new KafkaShuffleFetcher<>(
		sourceContext,
		assignedPartitionsWithInitialOffsets,
		watermarkStrategy,
		runtimeContext.getProcessingTimeService(),
		runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
		runtimeContext.getUserCodeClassLoader(),
		runtimeContext.getTaskNameWithSubtasks(),
		deserializer,
		properties,
		pollTimeout,
		runtimeContext.getMetricGroup(),
		consumerMetricGroup,
		useMetrics,
		typeSerializer,
		producerParallelism);
}
 
Example 16
Source Project: flink   Source File: CheckpointStatsTracker.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Register the exposed metrics.
 *
 * @param metricGroup Metric group to use for the metrics.
 */
private void registerMetrics(MetricGroup metricGroup) {
	metricGroup.gauge(NUMBER_OF_CHECKPOINTS_METRIC, new CheckpointsCounter());
	metricGroup.gauge(NUMBER_OF_IN_PROGRESS_CHECKPOINTS_METRIC, new InProgressCheckpointsCounter());
	metricGroup.gauge(NUMBER_OF_COMPLETED_CHECKPOINTS_METRIC, new CompletedCheckpointsCounter());
	metricGroup.gauge(NUMBER_OF_FAILED_CHECKPOINTS_METRIC, new FailedCheckpointsCounter());
	metricGroup.gauge(LATEST_RESTORED_CHECKPOINT_TIMESTAMP_METRIC, new LatestRestoredCheckpointTimestampGauge());
	metricGroup.gauge(LATEST_COMPLETED_CHECKPOINT_SIZE_METRIC, new LatestCompletedCheckpointSizeGauge());
	metricGroup.gauge(LATEST_COMPLETED_CHECKPOINT_DURATION_METRIC, new LatestCompletedCheckpointDurationGauge());
	metricGroup.gauge(LATEST_COMPLETED_CHECKPOINT_EXTERNAL_PATH_METRIC, new LatestCompletedCheckpointExternalPathGauge());
}
 
Example 17
Source Project: flink   Source File: FlinkKafkaConsumer09.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> assignedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {

	// make sure that auto commit is disabled when our offset commit mode is ON_CHECKPOINTS;
	// this overwrites whatever setting the user configured in the properties
	adjustAutoCommitConfig(properties, offsetCommitMode);

	// If a rateLimiter is set, then call rateLimiter.open() with the runtime context.
	if (rateLimiter != null) {
		rateLimiter.open(runtimeContext);
	}

	return new Kafka09Fetcher<>(
			sourceContext,
			assignedPartitionsWithInitialOffsets,
			watermarksPeriodic,
			watermarksPunctuated,
			runtimeContext.getProcessingTimeService(),
			runtimeContext.getExecutionConfig().getAutoWatermarkInterval(),
			runtimeContext.getUserCodeClassLoader(),
			runtimeContext.getTaskNameWithSubtasks(),
			deserializer,
			properties,
			pollTimeout,
			runtimeContext.getMetricGroup(),
			consumerMetricGroup,
			useMetrics,
			rateLimiter);
}
 
Example 18
Source Project: flink   Source File: AbstractStateBackend.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public abstract <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws IOException;
 
Example 19
@Override
protected AbstractFetcher<T, ?> createFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> thisSubtaskPartitionsWithStartOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		StreamingRuntimeContext runtimeContext,
		OffsetCommitMode offsetCommitMode,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	return fetcher;
}
 
Example 20
Source Project: flink   Source File: AbstractPrometheusReporter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void notifyOfAddedMetric(final Metric metric, final String metricName, final MetricGroup group) {

	List<String> dimensionKeys = new LinkedList<>();
	List<String> dimensionValues = new LinkedList<>();
	for (final Map.Entry<String, String> dimension : group.getAllVariables().entrySet()) {
		final String key = dimension.getKey();
		dimensionKeys.add(CHARACTER_FILTER.filterCharacters(key.substring(1, key.length() - 1)));
		dimensionValues.add(labelValueCharactersFilter.filterCharacters(dimension.getValue()));
	}

	final String scopedMetricName = getScopedName(metricName, group);
	final String helpString = metricName + " (scope: " + getLogicalScope(group) + ")";

	final Collector collector;
	Integer count = 0;

	synchronized (this) {
		if (collectorsWithCountByMetricName.containsKey(scopedMetricName)) {
			final AbstractMap.SimpleImmutableEntry<Collector, Integer> collectorWithCount = collectorsWithCountByMetricName.get(scopedMetricName);
			collector = collectorWithCount.getKey();
			count = collectorWithCount.getValue();
		} else {
			collector = createCollector(metric, dimensionKeys, dimensionValues, scopedMetricName, helpString);
			try {
				collector.register();
			} catch (Exception e) {
				log.warn("There was a problem registering metric {}.", metricName, e);
			}
		}
		addMetric(metric, dimensionValues, collector);
		collectorsWithCountByMetricName.put(scopedMetricName, new AbstractMap.SimpleImmutableEntry<>(collector, count + 1));
	}
}
 
Example 21
Source Project: flink   Source File: CollectionExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception {
	Operator<?> inputOp = sink.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The data sink " + sink.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> input = (List<IN>) execute(inputOp);
	
	@SuppressWarnings("unchecked")
	GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink;

	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
		
	if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}

	typedSink.executeOnCollections(input, ctx, executionConfig);
}
 
Example 22
Source Project: flink   Source File: NettyShuffleMetricFactory.java    License: Apache License 2.0 5 votes vote down vote up
public static void registerShuffleMetrics(
		MetricGroup metricGroup,
		NetworkBufferPool networkBufferPool) {
	checkNotNull(metricGroup);
	checkNotNull(networkBufferPool);

	//noinspection deprecation
	registerShuffleMetrics(METRIC_GROUP_NETWORK_DEPRECATED, metricGroup, networkBufferPool);
	registerShuffleMetrics(METRIC_GROUP_NETTY, metricGroup.addGroup(METRIC_GROUP_SHUFFLE), networkBufferPool);
}
 
Example 23
Source Project: flink   Source File: RocksDBNativeMetricMonitor.java    License: Apache License 2.0 5 votes vote down vote up
public RocksDBNativeMetricMonitor(
	@Nonnull RocksDBNativeMetricOptions options,
	@Nonnull MetricGroup metricGroup,
	@Nonnull RocksDB rocksDB
) {
	this.options = options;
	this.metricGroup = metricGroup;
	this.rocksDB = rocksDB;

	this.lock = new Object();
}
 
Example 24
Source Project: Flink-CEPplus   Source File: LatencyStats.java    License: Apache License 2.0 5 votes vote down vote up
@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base;
}
 
Example 25
Source Project: flink   Source File: StateBackendITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws IOException {
	throw new SuccessException();
}
 
Example 26
Source Project: Flink-CEPplus   Source File: CollectionExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private <IN1, IN2, OUT> List<OUT> executeBinaryOperator(DualInputOperator<?, ?, ?, ?> operator, int superStep) throws Exception {
	Operator<?> inputOp1 = operator.getFirstInput();
	Operator<?> inputOp2 = operator.getSecondInput();
	
	if (inputOp1 == null) {
		throw new InvalidProgramException("The binary operation " + operator.getName() + " has no first input.");
	}
	if (inputOp2 == null) {
		throw new InvalidProgramException("The binary operation " + operator.getName() + " has no second input.");
	}
	
	// compute inputs
	@SuppressWarnings("unchecked")
	List<IN1> inputData1 = (List<IN1>) execute(inputOp1, superStep);
	@SuppressWarnings("unchecked")
	List<IN2> inputData2 = (List<IN2>) execute(inputOp2, superStep);
	
	@SuppressWarnings("unchecked")
	DualInputOperator<IN1, IN2, OUT, ?> typedOp = (DualInputOperator<IN1, IN2, OUT, ?>) operator;
	
	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();

	if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
			new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
		
		for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
			List<?> bcData = execute(bcInputs.getValue());
			ctx.setBroadcastVariable(bcInputs.getKey(), bcData);
		}
	} else {
		ctx = null;
	}

	return typedOp.executeOnCollections(inputData1, inputData2, ctx, executionConfig);
}
 
Example 27
Source Project: flink   Source File: RocksDBNativeMetricMonitor.java    License: Apache License 2.0 5 votes vote down vote up
public RocksDBNativeMetricMonitor(
	@Nonnull RocksDBNativeMetricOptions options,
	@Nonnull MetricGroup metricGroup,
	@Nonnull RocksDB rocksDB
) {
	this.options = options;
	this.metricGroup = metricGroup;
	this.rocksDB = rocksDB;

	this.lock = new Object();
}
 
Example 28
Source Project: Flink-CEPplus   Source File: MetricRegistryImplTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void notifyOfAddedMetric(Metric metric, String metricName, MetricGroup group) {
	String expectedMetric = "A" + expectedDelimiter + "B" + expectedDelimiter + "C";
	assertEquals(expectedMetric, group.getMetricIdentifier(metricName, this));
	assertEquals(expectedMetric, group.getMetricIdentifier(metricName));
	numCorrectDelimitersForRegister++;
}
 
Example 29
Source Project: flink   Source File: AbstractRuntimeUDFContext.java    License: Apache License 2.0 5 votes vote down vote up
public AbstractRuntimeUDFContext(TaskInfo taskInfo,
									ClassLoader userCodeClassLoader,
									ExecutionConfig executionConfig,
									Map<String, Accumulator<?, ?>> accumulators,
									Map<String, Future<Path>> cpTasks,
									MetricGroup metrics) {
	this.taskInfo = checkNotNull(taskInfo);
	this.userCodeClassLoader = userCodeClassLoader;
	this.executionConfig = executionConfig;
	this.distributedCache = new DistributedCache(checkNotNull(cpTasks));
	this.accumulators = checkNotNull(accumulators);
	this.metrics = metrics;
}
 
Example 30
Source Project: flink   Source File: LatencyStats.java    License: Apache License 2.0 5 votes vote down vote up
@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base;
}