Java Code Examples for org.apache.flink.runtime.metrics.MetricRegistryConfiguration#fromConfiguration()

/**
 * Verifies that when attempting to create a group with the name of an existing one the existing one will be returned instead.
 */
@Test
public void testDuplicateGroupName() throws Exception {
	Configuration config = new Configuration();

	MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(config));

	MetricGroup root = new TaskManagerMetricGroup(registry, "host", "id");

	MetricGroup group1 = root.addGroup("group");
	MetricGroup group2 = root.addGroup("group");
	MetricGroup group3 = root.addGroup("group");
	Assert.assertTrue(group1 == group2 && group2 == group3);

	registry.shutdown().get();
}
 

@Test
public void testGenerateScopeWilcard() throws Exception {
	Configuration cfg = new Configuration();
	cfg.setString(MetricOptions.SCOPE_NAMING_TASK, "*.<task_attempt_id>.<subtask_index>");
	MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(cfg));

	AbstractID executionId = new AbstractID();

	TaskManagerMetricGroup tmGroup = new TaskManagerMetricGroup(registry, "theHostName", "test-tm-id");
	TaskManagerJobMetricGroup jmGroup = new TaskManagerJobMetricGroup(registry, tmGroup, new JobID(), "myJobName");

	TaskMetricGroup taskGroup = new TaskMetricGroup(
			registry, jmGroup, new JobVertexID(), executionId, "aTaskName", 13, 1);

	assertArrayEquals(
			new String[]{"theHostName", "taskmanager", "test-tm-id", "myJobName", executionId.toString(), "13"},
			taskGroup.getScopeComponents());

	assertEquals(
			"theHostName.taskmanager.test-tm-id.myJobName." + executionId + ".13.name",
			taskGroup.getMetricIdentifier("name"));
	registry.shutdown().get();
}
 

@Test
public void testScopeCachingForMultipleReporters() throws Exception {
	Configuration config = new Configuration();
	config.setString(MetricOptions.SCOPE_NAMING_TM, "A.B.C.D");
	config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter1.class.getName());
	config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "-");
	config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter2.class.getName());
	config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "!");

	MetricRegistryImpl testRegistry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(config));
	try {
		MetricGroup tmGroup = new TaskManagerMetricGroup(testRegistry, "host", "id");
		tmGroup.counter("1");
		assertEquals("Reporters were not properly instantiated", 2, testRegistry.getReporters().size());
		for (MetricReporter reporter : testRegistry.getReporters()) {
			ScopeCheckingTestReporter typedReporter = (ScopeCheckingTestReporter) reporter;
			if (typedReporter.failureCause != null) {
				throw typedReporter.failureCause;
			}
		}
	} finally {
		testRegistry.shutdown().get();
	}
}
 

@Test
public void testGenerateScopeCustom() throws Exception {
	Configuration cfg = new Configuration();
	cfg.setString(MetricOptions.SCOPE_NAMING_JM, "abc");
	cfg.setString(MetricOptions.SCOPE_NAMING_JM_JOB, "some-constant.<job_name>");
	MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(cfg));

	JobID jid = new JobID();

	JobManagerMetricGroup tmGroup = new JobManagerMetricGroup(registry, "theHostName");
	JobMetricGroup jmGroup = new JobManagerJobMetricGroup(registry, tmGroup, jid, "myJobName");

	assertArrayEquals(
			new String[] { "some-constant", "myJobName" },
			jmGroup.getScopeComponents());

	assertEquals(
			"some-constant.myJobName.name",
			jmGroup.getMetricIdentifier("name"));

	registry.shutdown().get();
}
 

@Test
public void testGenerateScopeCustom() throws Exception {
	Configuration cfg = new Configuration();
	cfg.setString(MetricOptions.SCOPE_NAMING_JM, "abc");
	cfg.setString(MetricOptions.SCOPE_NAMING_JM_JOB, "some-constant.<job_name>");
	MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(cfg));

	JobID jid = new JobID();

	JobManagerMetricGroup tmGroup = new JobManagerMetricGroup(registry, "theHostName");
	JobMetricGroup jmGroup = new JobManagerJobMetricGroup(registry, tmGroup, jid, "myJobName");

	assertArrayEquals(
			new String[] { "some-constant", "myJobName" },
			jmGroup.getScopeComponents());

	assertEquals(
			"some-constant.myJobName.name",
			jmGroup.getMetricIdentifier("name"));

	registry.shutdown().get();
}
 

@Test
public void testOperatorNameTruncation() throws Exception {
	Configuration cfg = new Configuration();
	cfg.setString(MetricOptions.SCOPE_NAMING_OPERATOR, ScopeFormat.SCOPE_OPERATOR_NAME);
	MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(cfg));
	TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id");
	TaskManagerJobMetricGroup job = new TaskManagerJobMetricGroup(registry, tm, new JobID(), "jobname");
	TaskMetricGroup taskMetricGroup = new TaskMetricGroup(registry, job, new JobVertexID(), new AbstractID(), "task", 0, 0);

	String originalName = new String(new char[100]).replace("\0", "-");
	OperatorMetricGroup operatorMetricGroup = taskMetricGroup.getOrAddOperator(originalName);

	String storedName = operatorMetricGroup.getScopeComponents()[0];
	Assert.assertEquals(TaskMetricGroup.METRICS_OPERATOR_NAME_MAX_LENGTH, storedName.length());
	Assert.assertEquals(originalName.substring(0, TaskMetricGroup.METRICS_OPERATOR_NAME_MAX_LENGTH), storedName);
	registry.shutdown().get();
}
 

@Test
public void testGenerateScopeWilcard() throws Exception {
	Configuration cfg = new Configuration();
	cfg.setString(MetricOptions.SCOPE_NAMING_TASK, "*.<task_attempt_id>.<subtask_index>");
	MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(cfg));

	AbstractID executionId = new AbstractID();

	TaskManagerMetricGroup tmGroup = new TaskManagerMetricGroup(registry, "theHostName", "test-tm-id");
	TaskManagerJobMetricGroup jmGroup = new TaskManagerJobMetricGroup(registry, tmGroup, new JobID(), "myJobName");

	TaskMetricGroup taskGroup = new TaskMetricGroup(
			registry, jmGroup, new JobVertexID(), executionId, "aTaskName", 13, 1);

	assertArrayEquals(
			new String[]{"theHostName", "taskmanager", "test-tm-id", "myJobName", executionId.toString(), "13"},
			taskGroup.getScopeComponents());

	assertEquals(
			"theHostName.taskmanager.test-tm-id.myJobName." + executionId + ".13.name",
			taskGroup.getMetricIdentifier("name"));
	registry.shutdown().get();
}
 

@Test
public void testGenerateScopeCustomWildcard() throws Exception {
	Configuration cfg = new Configuration();
	cfg.setString(MetricOptions.SCOPE_NAMING_TM, "peter.<tm_id>");
	cfg.setString(MetricOptions.SCOPE_NAMING_TM_JOB, "*.some-constant.<job_id>");
	MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(cfg));

	JobID jid = new JobID();

	TaskManagerMetricGroup tmGroup = new TaskManagerMetricGroup(registry, "theHostName", "test-tm-id");
	JobMetricGroup jmGroup = new TaskManagerJobMetricGroup(registry, tmGroup, jid, "myJobName");

	assertArrayEquals(
			new String[] { "peter", "test-tm-id", "some-constant", jid.toString() },
			jmGroup.getScopeComponents());

	assertEquals(
			"peter.test-tm-id.some-constant." + jid + ".name",
			jmGroup.getMetricIdentifier("name"));
	registry.shutdown().get();
}
 

@Test
public void testOperatorNameTruncation() throws Exception {
	Configuration cfg = new Configuration();
	cfg.setString(MetricOptions.SCOPE_NAMING_OPERATOR, ScopeFormat.SCOPE_OPERATOR_NAME);
	MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(cfg));
	TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id");
	TaskManagerJobMetricGroup job = new TaskManagerJobMetricGroup(registry, tm, new JobID(), "jobname");
	TaskMetricGroup taskMetricGroup = new TaskMetricGroup(registry, job, new JobVertexID(), new AbstractID(), "task", 0, 0);

	String originalName = new String(new char[100]).replace("\0", "-");
	OperatorMetricGroup operatorMetricGroup = taskMetricGroup.getOrAddOperator(originalName);

	String storedName = operatorMetricGroup.getScopeComponents()[0];
	Assert.assertEquals(TaskMetricGroup.METRICS_OPERATOR_NAME_MAX_LENGTH, storedName.length());
	Assert.assertEquals(originalName.substring(0, TaskMetricGroup.METRICS_OPERATOR_NAME_MAX_LENGTH), storedName);
	registry.shutdown().get();
}
 

@Test
public void testGenerateScopeWilcard() throws Exception {
	Configuration cfg = new Configuration();
	cfg.setString(MetricOptions.SCOPE_NAMING_TASK, "*.<task_attempt_id>.<subtask_index>");
	MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(cfg));

	AbstractID executionId = new AbstractID();

	TaskManagerMetricGroup tmGroup = new TaskManagerMetricGroup(registry, "theHostName", "test-tm-id");
	TaskManagerJobMetricGroup jmGroup = new TaskManagerJobMetricGroup(registry, tmGroup, new JobID(), "myJobName");

	TaskMetricGroup taskGroup = new TaskMetricGroup(
			registry, jmGroup, new JobVertexID(), executionId, "aTaskName", 13, 1);

	assertArrayEquals(
			new String[]{"theHostName", "taskmanager", "test-tm-id", "myJobName", executionId.toString(), "13"},
			taskGroup.getScopeComponents());

	assertEquals(
			"theHostName.taskmanager.test-tm-id.myJobName." + executionId + ".13.name",
			taskGroup.getMetricIdentifier("name"));
	registry.shutdown().get();
}
 

/**
 * Verifies that calling {@link AbstractMetricGroup#getLogicalScope(CharacterFilter, char, int)} on {@link GenericValueMetricGroup}
 * should ignore value as well.
 */
@Test
public void testLogicalScopeShouldIgnoreValueGroupName() throws Exception {
	Configuration config = new Configuration();
	config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter.class.getName());

	MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(config));
	try {
		GenericMetricGroup root = new GenericMetricGroup(registry, new DummyAbstractMetricGroup(registry), "root");

		String key = "key";
		String value = "value";

		MetricGroup group = root.addGroup(key, value);

		String logicalScope = ((AbstractMetricGroup) group)
			.getLogicalScope(new DummyCharacterFilter(), registry.getDelimiter(), 0);
		assertThat("Key is missing from logical scope.", logicalScope, containsString(key));
		assertThat("Value is present in logical scope.", logicalScope, not(containsString(value)));
	} finally {
		registry.shutdown().get();
	}
}
 

@Test
public void testScopeGenerationWithoutReporters() throws Exception {
	Configuration config = new Configuration();
	config.setString(MetricOptions.SCOPE_NAMING_TM, "A.B.C.D");
	MetricRegistryImpl testRegistry = new MetricRegistryImpl(
		MetricRegistryConfiguration.fromConfiguration(config));

	try {
		TaskManagerMetricGroup group = new TaskManagerMetricGroup(testRegistry, "host", "id");
		assertEquals("MetricReporters list should be empty", 0, testRegistry.getReporters().size());

		// default delimiter should be used
		assertEquals("A.B.X.D.1", group.getMetricIdentifier("1", FILTER_C));
		// no caching should occur
		assertEquals("A.X.C.D.1", group.getMetricIdentifier("1", FILTER_B));
		// invalid reporter indices do not throw errors
		assertEquals("A.X.C.D.1", group.getMetricIdentifier("1", FILTER_B, -1));
		assertEquals("A.X.C.D.1", group.getMetricIdentifier("1", FILTER_B, 2));
	} finally {
		testRegistry.shutdown().get();
	}
}
 

@Test
public void testGenerateScopeCustom() throws Exception {
	Configuration cfg = new Configuration();
	cfg.setString(MetricOptions.SCOPE_NAMING_JM, "constant.<host>.foo.<host>");
	MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(cfg));

	JobManagerMetricGroup group = new JobManagerMetricGroup(registry, "host");

	assertArrayEquals(new String[]{"constant", "host", "foo", "host"}, group.getScopeComponents());
	assertEquals("constant.host.foo.host.name", group.getMetricIdentifier("name"));

	registry.shutdown().get();
}
 

@BeforeClass
public static void setUp() {
	Configuration configuration = new Configuration();
	configuration.setString(MetricOptions.SCOPE_NAMING_TASK, "<host>.<tm_id>.<job_name>");

	registry = new MetricRegistryImpl(
		MetricRegistryConfiguration.fromConfiguration(configuration),
		Collections.singletonList(ReporterSetup.forReporter("slf4j", new Slf4jReporter())));
	delimiter = registry.getDelimiter();

	taskMetricGroup = new TaskManagerMetricGroup(registry, HOST_NAME, TASK_MANAGER_ID)
		.addTaskForJob(new JobID(), JOB_NAME, new JobVertexID(), new ExecutionAttemptID(), TASK_NAME, 0, 0);
	reporter = (Slf4jReporter) registry.getReporters().get(0);
}
 

/**
 * Factory method to create the metric registry for the mini cluster.
 *
 * @param config The configuration of the mini cluster
 */
protected MetricRegistryImpl createMetricRegistry(Configuration config) {
	return new MetricRegistryImpl(
		MetricRegistryConfiguration.fromConfiguration(config),
		ReporterSetup.fromConfiguration(config));
}
 

public TaskManagerRunner(Configuration configuration, ResourceID resourceId) throws Exception {
	this.configuration = checkNotNull(configuration);
	this.resourceId = checkNotNull(resourceId);

	timeout = AkkaUtils.getTimeoutAsTime(configuration);

	this.executor = java.util.concurrent.Executors.newScheduledThreadPool(
		Hardware.getNumberCPUCores(),
		new ExecutorThreadFactory("taskmanager-future"));

	highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
		configuration,
		executor,
		HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION);

	rpcService = createRpcService(configuration, highAvailabilityServices);

	HeartbeatServices heartbeatServices = HeartbeatServices.fromConfiguration(configuration);

	metricRegistry = new MetricRegistryImpl(
		MetricRegistryConfiguration.fromConfiguration(configuration),
		ReporterSetup.fromConfiguration(configuration));

	final RpcService metricQueryServiceRpcService = MetricUtils.startMetricsRpcService(configuration, rpcService.getAddress());
	metricRegistry.startQueryService(metricQueryServiceRpcService, resourceId);

	blobCacheService = new BlobCacheService(
		configuration, highAvailabilityServices.createBlobStore(), null
	);

	taskManager = startTaskManager(
		this.configuration,
		this.resourceId,
		rpcService,
		highAvailabilityServices,
		heartbeatServices,
		metricRegistry,
		blobCacheService,
		false,
		this);

	this.terminationFuture = new CompletableFuture<>();
	this.shutdown = false;

	MemoryLogger.startIfConfigured(LOG, configuration, terminationFuture);
}
 

/**
 * Verifies that multiple JMXReporters can be started on the same machine and register metrics at the MBeanServer.
 *
 * @throws Exception if the attribute/mbean could not be found or the test is broken
 */
@Test
public void testPortConflictHandling() throws Exception {
	Configuration cfg = new Configuration();

	cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName());
	cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1.port", "9020-9035");

	cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName());
	cfg.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2.port", "9020-9035");

	MetricRegistryImpl reg = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(cfg));

	TaskManagerMetricGroup mg = new TaskManagerMetricGroup(reg, "host", "tm");

	List<MetricReporter> reporters = reg.getReporters();

	assertTrue(reporters.size() == 2);

	MetricReporter rep1 = reporters.get(0);
	MetricReporter rep2 = reporters.get(1);

	Gauge<Integer> g1 = new Gauge<Integer>() {
		@Override
		public Integer getValue() {
			return 1;
		}
	};
	Gauge<Integer> g2 = new Gauge<Integer>() {
		@Override
		public Integer getValue() {
			return 2;
		}
	};

	rep1.notifyOfAddedMetric(g1, "rep1", new FrontMetricGroup<>(0, new TaskManagerMetricGroup(reg, "host", "tm")));
	rep2.notifyOfAddedMetric(g2, "rep2", new FrontMetricGroup<>(0, new TaskManagerMetricGroup(reg, "host", "tm")));

	MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer();

	ObjectName objectName1 = new ObjectName(JMX_DOMAIN_PREFIX + "taskmanager.rep1", JMXReporter.generateJmxTable(mg.getAllVariables()));
	ObjectName objectName2 = new ObjectName(JMX_DOMAIN_PREFIX + "taskmanager.rep2", JMXReporter.generateJmxTable(mg.getAllVariables()));

	assertEquals(1, mBeanServer.getAttribute(objectName1, "Value"));
	assertEquals(2, mBeanServer.getAttribute(objectName2, "Value"));

	rep1.notifyOfRemovedMetric(g1, "rep1", null);
	rep1.notifyOfRemovedMetric(g2, "rep2", null);

	mg.close();
	reg.shutdown().get();
}
 

protected MetricRegistryImpl createMetricRegistry(Configuration configuration) {
	return new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(configuration));
}
 

public TaskManagerRunner(Configuration configuration, ResourceID resourceId) throws Exception {
	this.configuration = checkNotNull(configuration);
	this.resourceId = checkNotNull(resourceId);

	timeout = AkkaUtils.getTimeoutAsTime(configuration);

	this.executor = java.util.concurrent.Executors.newScheduledThreadPool(
		Hardware.getNumberCPUCores(),
		new ExecutorThreadFactory("taskmanager-future"));

	highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
		configuration,
		executor,
		HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION);

	rpcService = createRpcService(configuration, highAvailabilityServices);
	metricQueryServiceActorSystem = MetricUtils.startMetricsActorSystem(configuration, rpcService.getAddress(), LOG);

	HeartbeatServices heartbeatServices = HeartbeatServices.fromConfiguration(configuration);

	metricRegistry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(configuration));

	// TODO: Temporary hack until the MetricQueryService has been ported to RpcEndpoint
	metricRegistry.startQueryService(metricQueryServiceActorSystem, resourceId);

	blobCacheService = new BlobCacheService(
		configuration, highAvailabilityServices.createBlobStore(), null
	);

	taskManager = startTaskManager(
		this.configuration,
		this.resourceId,
		rpcService,
		highAvailabilityServices,
		heartbeatServices,
		metricRegistry,
		blobCacheService,
		false,
		this);

	this.terminationFuture = new CompletableFuture<>();
	this.shutdown = false;

	MemoryLogger.startIfConfigured(LOG, configuration, metricQueryServiceActorSystem);
}
 

/**
 * Factory method to create the metric registry for the mini cluster.
 *
 * @param config The configuration of the mini cluster
 */
protected MetricRegistryImpl createMetricRegistry(Configuration config) {
	return new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(config));
}