org.apache.flink.runtime.metrics.dump.MetricDump Java Examples

The following examples show how to use org.apache.flink.runtime.metrics.dump.MetricDump. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: MetricsHandlerTestBase.java From flink with Apache License 2.0

6 votes

@Before
public void setUp() {
	MockitoAnnotations.initMocks(this);

	this.leaderRetriever = new GatewayRetriever<DispatcherGateway>() {
		@Override
		public CompletableFuture<DispatcherGateway> getFuture() {
			return CompletableFuture.completedFuture(mockDispatcherGateway);
		}
	};
	this.pathParameters = getPathParameters();
	this.metricsHandler = getMetricsHandler();

	final MetricStore metricStore = new MetricStore();
	metricStore.add(new MetricDump.CounterDump(getQueryScopeInfo(), TEST_METRIC_NAME,
		TEST_METRIC_VALUE));
	when(mockMetricFetcher.getMetricStore()).thenReturn(metricStore);
}

Example #2

Source File: MetricsHandlerTestBase.java From flink with Apache License 2.0

6 votes

@Before
public void setUp() {
	MockitoAnnotations.initMocks(this);

	this.leaderRetriever = new GatewayRetriever<DispatcherGateway>() {
		@Override
		public CompletableFuture<DispatcherGateway> getFuture() {
			return CompletableFuture.completedFuture(mockDispatcherGateway);
		}
	};
	this.pathParameters = getPathParameters();
	this.metricsHandler = getMetricsHandler();

	final MetricStore metricStore = new MetricStore();
	metricStore.add(new MetricDump.CounterDump(getQueryScopeInfo(), TEST_METRIC_NAME,
		TEST_METRIC_VALUE));
	when(mockMetricFetcher.getMetricStore()).thenReturn(metricStore);
}

Example #3

Source File: AbstractMetricsHandlerTest.java From flink with Apache License 2.0

6 votes

@Before
public void setUp() {
	MockitoAnnotations.initMocks(this);

	final MetricStore metricStore = new MetricStore();
	metricStore.add(new MetricDump.CounterDump(
		new QueryScopeInfo.JobManagerQueryScopeInfo(),
		TEST_METRIC_NAME,
		TEST_METRIC_VALUE));

	when(mockMetricFetcher.getMetricStore()).thenReturn(metricStore);

	testMetricsHandler = new TestMetricsHandler(
		new GatewayRetriever<DispatcherGateway>() {
			@Override
			public CompletableFuture<DispatcherGateway> getFuture() {
				return CompletableFuture.completedFuture(mockDispatcherGateway);
			}
		},
		Time.milliseconds(50),
		Collections.emptyMap(),
		new TestMetricsHeaders(),
		mockMetricFetcher);
}

Example #4

Source File: AggregatingJobsMetricsHandlerTest.java From flink with Apache License 2.0

6 votes

@Override
protected Collection<MetricDump> getMetricDumps() {
	Collection<MetricDump> dumps = new ArrayList<>(3);
	QueryScopeInfo.JobQueryScopeInfo job = new QueryScopeInfo.JobQueryScopeInfo(JOB_ID_1.toString(), "abc");
	MetricDump.CounterDump cd1 = new MetricDump.CounterDump(job, "metric1", 1);
	dumps.add(cd1);

	QueryScopeInfo.JobQueryScopeInfo job2 = new QueryScopeInfo.JobQueryScopeInfo(JOB_ID_2.toString(), "abc");
	MetricDump.CounterDump cd2 = new MetricDump.CounterDump(job2, "metric1", 3);
	dumps.add(cd2);

	QueryScopeInfo.JobQueryScopeInfo job3 = new QueryScopeInfo.JobQueryScopeInfo(JOB_ID_3.toString(), "abc");
	MetricDump.CounterDump cd3 = new MetricDump.CounterDump(job3, "metric2", 5);
	dumps.add(cd3);
	return dumps;
}

Example #5

Source File: AggregatingMetricsHandlerTestBase.java From flink with Apache License 2.0

6 votes

@Before
public void setUp() throws Exception {
	MetricFetcher fetcher = new MetricFetcherImpl<RestfulGateway>(
		mock(GatewayRetriever.class),
		mock(MetricQueryServiceRetriever.class),
		Executors.directExecutor(),
		TestingUtils.TIMEOUT(),
		MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());
	store = fetcher.getMetricStore();

	Collection<MetricDump> metricDumps = getMetricDumps();
	for (MetricDump dump : metricDumps) {
		store.add(dump);
	}

	handler = getHandler(
		LEADER_RETRIEVER,
		TIMEOUT,
		TEST_HEADERS,
		EXECUTOR,
		fetcher);
	pathParameters = getPathParameters();
}

Example #6

Source File: AggregatingMetricsHandlerTestBase.java From flink with Apache License 2.0

6 votes

@Before
public void setUp() throws Exception {
	MetricFetcher fetcher = new MetricFetcherImpl<RestfulGateway>(
		mock(GatewayRetriever.class),
		mock(MetricQueryServiceRetriever.class),
		Executors.directExecutor(),
		TestingUtils.TIMEOUT(),
		MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());
	store = fetcher.getMetricStore();

	Collection<MetricDump> metricDumps = getMetricDumps();
	for (MetricDump dump : metricDumps) {
		store.add(dump);
	}

	handler = getHandler(
		LEADER_RETRIEVER,
		TIMEOUT,
		TEST_HEADERS,
		EXECUTOR,
		fetcher);
	pathParameters = getPathParameters();
}

Example #7

Source File: AggregatingSubtasksMetricsHandlerTest.java From flink with Apache License 2.0

6 votes

@Override
protected Collection<MetricDump> getMetricDumps() {
	Collection<MetricDump> dumps = new ArrayList<>(3);
	QueryScopeInfo.TaskQueryScopeInfo task1 = new QueryScopeInfo.TaskQueryScopeInfo(JOB_ID.toString(), TASK_ID.toString(), 1, "abc");
	MetricDump.CounterDump cd1 = new MetricDump.CounterDump(task1, "metric1", 1);
	dumps.add(cd1);

	QueryScopeInfo.TaskQueryScopeInfo task2 = new QueryScopeInfo.TaskQueryScopeInfo(JOB_ID.toString(), TASK_ID.toString(), 2, "abc");
	MetricDump.CounterDump cd2 = new MetricDump.CounterDump(task2, "metric1", 3);
	dumps.add(cd2);

	QueryScopeInfo.TaskQueryScopeInfo task3 = new QueryScopeInfo.TaskQueryScopeInfo(JOB_ID.toString(), TASK_ID.toString(), 3, "abc");
	MetricDump.CounterDump cd3 = new MetricDump.CounterDump(task3, "metric2", 5);
	dumps.add(cd3);

	return dumps;
}

Example #8

Source File: AggregatingTaskManagersMetricsHandlerTest.java From flink with Apache License 2.0

6 votes

@Override
protected Collection<MetricDump> getMetricDumps() {
	Collection<MetricDump> dumps = new ArrayList<>(3);
	QueryScopeInfo.TaskManagerQueryScopeInfo tm1 = new QueryScopeInfo.TaskManagerQueryScopeInfo(TM_ID_1.toString(), "abc");
	MetricDump.CounterDump cd1 = new MetricDump.CounterDump(tm1, "metric1", 1);
	dumps.add(cd1);

	QueryScopeInfo.TaskManagerQueryScopeInfo tm2 = new QueryScopeInfo.TaskManagerQueryScopeInfo(TM_ID_2.toString(), "abc");
	MetricDump.CounterDump cd2 = new MetricDump.CounterDump(tm2, "metric1", 3);
	dumps.add(cd2);

	QueryScopeInfo.TaskManagerQueryScopeInfo tm3 = new QueryScopeInfo.TaskManagerQueryScopeInfo(TM_ID_3.toString(), "abc");
	MetricDump.CounterDump cd3 = new MetricDump.CounterDump(tm3, "metric2", 5);
	dumps.add(cd3);

	return dumps;
}

Example #9

Source File: MetricStoreTest.java From flink with Apache License 2.0

6 votes

@Test
public void testMalformedNameHandling() {
	MetricStore store = new MetricStore();
	//-----verify that no exceptions are thrown

	// null
	store.add(null);
	// empty name
	QueryScopeInfo.JobManagerQueryScopeInfo info = new QueryScopeInfo.JobManagerQueryScopeInfo("");
	MetricDump.CounterDump cd = new MetricDump.CounterDump(info, "", 0);
	store.add(cd);

	//-----verify that no side effects occur
	assertEquals(0, store.getJobManager().metrics.size());
	assertEquals(0, store.getTaskManagers().size());
	assertEquals(0, store.getJobs().size());
}

Example #10

Source File: AggregatingJobsMetricsHandlerTest.java From flink with Apache License 2.0

6 votes

@Override
protected Collection<MetricDump> getMetricDumps() {
	Collection<MetricDump> dumps = new ArrayList<>(3);
	QueryScopeInfo.JobQueryScopeInfo job = new QueryScopeInfo.JobQueryScopeInfo(JOB_ID_1.toString(), "abc");
	MetricDump.CounterDump cd1 = new MetricDump.CounterDump(job, "metric1", 1);
	dumps.add(cd1);

	QueryScopeInfo.JobQueryScopeInfo job2 = new QueryScopeInfo.JobQueryScopeInfo(JOB_ID_2.toString(), "abc");
	MetricDump.CounterDump cd2 = new MetricDump.CounterDump(job2, "metric1", 3);
	dumps.add(cd2);

	QueryScopeInfo.JobQueryScopeInfo job3 = new QueryScopeInfo.JobQueryScopeInfo(JOB_ID_3.toString(), "abc");
	MetricDump.CounterDump cd3 = new MetricDump.CounterDump(job3, "metric2", 5);
	dumps.add(cd3);
	return dumps;
}

Example #11

Source File: AbstractMetricsHandlerTest.java From flink with Apache License 2.0

6 votes

@Before
public void setUp() {
	MockitoAnnotations.initMocks(this);

	final MetricStore metricStore = new MetricStore();
	metricStore.add(new MetricDump.CounterDump(
		new QueryScopeInfo.JobManagerQueryScopeInfo(),
		TEST_METRIC_NAME,
		TEST_METRIC_VALUE));

	when(mockMetricFetcher.getMetricStore()).thenReturn(metricStore);

	testMetricsHandler = new TestMetricsHandler(
		new GatewayRetriever<DispatcherGateway>() {
			@Override
			public CompletableFuture<DispatcherGateway> getFuture() {
				return CompletableFuture.completedFuture(mockDispatcherGateway);
			}
		},
		Time.milliseconds(50),
		Collections.emptyMap(),
		new TestMetricsHeaders(),
		mockMetricFetcher);
}

Example #12

Source File: MetricStoreTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testMalformedNameHandling() {
	MetricStore store = new MetricStore();
	//-----verify that no exceptions are thrown

	// null
	store.add(null);
	// empty name
	QueryScopeInfo.JobManagerQueryScopeInfo info = new QueryScopeInfo.JobManagerQueryScopeInfo("");
	MetricDump.CounterDump cd = new MetricDump.CounterDump(info, "", 0);
	store.add(cd);

	//-----verify that no side effects occur
	assertEquals(0, store.getJobManager().metrics.size());
	assertEquals(0, store.getTaskManagers().size());
	assertEquals(0, store.getJobs().size());
}

Example #13

Source File: MetricsHandlerTestBase.java From Flink-CEPplus with Apache License 2.0

6 votes

@Before
public void setUp() {
	MockitoAnnotations.initMocks(this);

	this.leaderRetriever = new GatewayRetriever<DispatcherGateway>() {
		@Override
		public CompletableFuture<DispatcherGateway> getFuture() {
			return CompletableFuture.completedFuture(mockDispatcherGateway);
		}
	};
	this.pathParameters = getPathParameters();
	this.metricsHandler = getMetricsHandler();

	final MetricStore metricStore = new MetricStore();
	metricStore.add(new MetricDump.CounterDump(getQueryScopeInfo(), TEST_METRIC_NAME,
		TEST_METRIC_VALUE));
	when(mockMetricFetcher.getMetricStore()).thenReturn(metricStore);
}

Example #14

Source File: AbstractMetricsHandlerTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Before
public void setUp() {
	MockitoAnnotations.initMocks(this);

	final MetricStore metricStore = new MetricStore();
	metricStore.add(new MetricDump.CounterDump(
		new QueryScopeInfo.JobManagerQueryScopeInfo(),
		TEST_METRIC_NAME,
		TEST_METRIC_VALUE));

	when(mockMetricFetcher.getMetricStore()).thenReturn(metricStore);

	testMetricsHandler = new TestMetricsHandler(
		new GatewayRetriever<DispatcherGateway>() {
			@Override
			public CompletableFuture<DispatcherGateway> getFuture() {
				return CompletableFuture.completedFuture(mockDispatcherGateway);
			}
		},
		Time.milliseconds(50),
		Collections.emptyMap(),
		new TestMetricsHeaders(),
		mockMetricFetcher);
}

Example #15

Source File: AggregatingJobsMetricsHandlerTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
protected Collection<MetricDump> getMetricDumps() {
	Collection<MetricDump> dumps = new ArrayList<>(3);
	QueryScopeInfo.JobQueryScopeInfo job = new QueryScopeInfo.JobQueryScopeInfo(JOB_ID_1.toString(), "abc");
	MetricDump.CounterDump cd1 = new MetricDump.CounterDump(job, "metric1", 1);
	dumps.add(cd1);

	QueryScopeInfo.JobQueryScopeInfo job2 = new QueryScopeInfo.JobQueryScopeInfo(JOB_ID_2.toString(), "abc");
	MetricDump.CounterDump cd2 = new MetricDump.CounterDump(job2, "metric1", 3);
	dumps.add(cd2);

	QueryScopeInfo.JobQueryScopeInfo job3 = new QueryScopeInfo.JobQueryScopeInfo(JOB_ID_3.toString(), "abc");
	MetricDump.CounterDump cd3 = new MetricDump.CounterDump(job3, "metric2", 5);
	dumps.add(cd3);
	return dumps;
}

Example #16

Source File: MetricStoreTest.java From flink with Apache License 2.0

6 votes

@Test
public void testMalformedNameHandling() {
	MetricStore store = new MetricStore();
	//-----verify that no exceptions are thrown

	// null
	store.add(null);
	// empty name
	QueryScopeInfo.JobManagerQueryScopeInfo info = new QueryScopeInfo.JobManagerQueryScopeInfo("");
	MetricDump.CounterDump cd = new MetricDump.CounterDump(info, "", 0);
	store.add(cd);

	//-----verify that no side effects occur
	assertEquals(0, store.getJobManager().metrics.size());
	assertEquals(0, store.getTaskManagers().size());
	assertEquals(0, store.getJobs().size());
}

Example #17

Source File: AggregatingMetricsHandlerTestBase.java From Flink-CEPplus with Apache License 2.0

6 votes

@Before
public void setUp() throws Exception {
	MetricFetcher fetcher = new MetricFetcherImpl<RestfulGateway>(
		mock(GatewayRetriever.class),
		mock(MetricQueryServiceRetriever.class),
		Executors.directExecutor(),
		TestingUtils.TIMEOUT(),
		MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());
	store = fetcher.getMetricStore();

	Collection<MetricDump> metricDumps = getMetricDumps();
	for (MetricDump dump : metricDumps) {
		store.add(dump);
	}

	handler = getHandler(
		LEADER_RETRIEVER,
		TIMEOUT,
		TEST_HEADERS,
		EXECUTOR,
		fetcher);
	pathParameters = getPathParameters();
}

Example #18

Source File: AggregatingSubtasksMetricsHandlerTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
protected Collection<MetricDump> getMetricDumps() {
	Collection<MetricDump> dumps = new ArrayList<>(3);
	QueryScopeInfo.TaskQueryScopeInfo task1 = new QueryScopeInfo.TaskQueryScopeInfo(JOB_ID.toString(), TASK_ID.toString(), 1, "abc");
	MetricDump.CounterDump cd1 = new MetricDump.CounterDump(task1, "metric1", 1);
	dumps.add(cd1);

	QueryScopeInfo.TaskQueryScopeInfo task2 = new QueryScopeInfo.TaskQueryScopeInfo(JOB_ID.toString(), TASK_ID.toString(), 2, "abc");
	MetricDump.CounterDump cd2 = new MetricDump.CounterDump(task2, "metric1", 3);
	dumps.add(cd2);

	QueryScopeInfo.TaskQueryScopeInfo task3 = new QueryScopeInfo.TaskQueryScopeInfo(JOB_ID.toString(), TASK_ID.toString(), 3, "abc");
	MetricDump.CounterDump cd3 = new MetricDump.CounterDump(task3, "metric2", 5);
	dumps.add(cd3);

	return dumps;
}

Example #19

Source File: AggregatingTaskManagersMetricsHandlerTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
protected Collection<MetricDump> getMetricDumps() {
	Collection<MetricDump> dumps = new ArrayList<>(3);
	QueryScopeInfo.TaskManagerQueryScopeInfo tm1 = new QueryScopeInfo.TaskManagerQueryScopeInfo(TM_ID_1.toString(), "abc");
	MetricDump.CounterDump cd1 = new MetricDump.CounterDump(tm1, "metric1", 1);
	dumps.add(cd1);

	QueryScopeInfo.TaskManagerQueryScopeInfo tm2 = new QueryScopeInfo.TaskManagerQueryScopeInfo(TM_ID_2.toString(), "abc");
	MetricDump.CounterDump cd2 = new MetricDump.CounterDump(tm2, "metric1", 3);
	dumps.add(cd2);

	QueryScopeInfo.TaskManagerQueryScopeInfo tm3 = new QueryScopeInfo.TaskManagerQueryScopeInfo(TM_ID_3.toString(), "abc");
	MetricDump.CounterDump cd3 = new MetricDump.CounterDump(tm3, "metric2", 5);
	dumps.add(cd3);

	return dumps;
}

Example #20

Source File: AggregatingSubtasksMetricsHandlerTest.java From flink with Apache License 2.0

6 votes

@Override
protected Collection<MetricDump> getMetricDumps() {
	Collection<MetricDump> dumps = new ArrayList<>(3);
	QueryScopeInfo.TaskQueryScopeInfo task1 = new QueryScopeInfo.TaskQueryScopeInfo(JOB_ID.toString(), TASK_ID.toString(), 1, "abc");
	MetricDump.CounterDump cd1 = new MetricDump.CounterDump(task1, "metric1", 1);
	dumps.add(cd1);

	QueryScopeInfo.TaskQueryScopeInfo task2 = new QueryScopeInfo.TaskQueryScopeInfo(JOB_ID.toString(), TASK_ID.toString(), 2, "abc");
	MetricDump.CounterDump cd2 = new MetricDump.CounterDump(task2, "metric1", 3);
	dumps.add(cd2);

	QueryScopeInfo.TaskQueryScopeInfo task3 = new QueryScopeInfo.TaskQueryScopeInfo(JOB_ID.toString(), TASK_ID.toString(), 3, "abc");
	MetricDump.CounterDump cd3 = new MetricDump.CounterDump(task3, "metric2", 5);
	dumps.add(cd3);

	return dumps;
}

Example #21

Source File: AggregatingTaskManagersMetricsHandlerTest.java From flink with Apache License 2.0

6 votes

@Override
protected Collection<MetricDump> getMetricDumps() {
	Collection<MetricDump> dumps = new ArrayList<>(3);
	QueryScopeInfo.TaskManagerQueryScopeInfo tm1 = new QueryScopeInfo.TaskManagerQueryScopeInfo(TM_ID_1.toString(), "abc");
	MetricDump.CounterDump cd1 = new MetricDump.CounterDump(tm1, "metric1", 1);
	dumps.add(cd1);

	QueryScopeInfo.TaskManagerQueryScopeInfo tm2 = new QueryScopeInfo.TaskManagerQueryScopeInfo(TM_ID_2.toString(), "abc");
	MetricDump.CounterDump cd2 = new MetricDump.CounterDump(tm2, "metric1", 3);
	dumps.add(cd2);

	QueryScopeInfo.TaskManagerQueryScopeInfo tm3 = new QueryScopeInfo.TaskManagerQueryScopeInfo(TM_ID_3.toString(), "abc");
	MetricDump.CounterDump cd3 = new MetricDump.CounterDump(tm3, "metric2", 5);
	dumps.add(cd3);

	return dumps;
}

Example #22

Source File: MetricStore.java From flink with Apache License 2.0

5 votes

private void addMetric(Map<String, String> target, String name, MetricDump metric) {
	switch (metric.getCategory()) {
		case METRIC_CATEGORY_COUNTER:
			MetricDump.CounterDump counter = (MetricDump.CounterDump) metric;
			target.put(name, String.valueOf(counter.count));
			break;
		case METRIC_CATEGORY_GAUGE:
			MetricDump.GaugeDump gauge = (MetricDump.GaugeDump) metric;
			target.put(name, gauge.value);
			break;
		case METRIC_CATEGORY_HISTOGRAM:
			MetricDump.HistogramDump histogram = (MetricDump.HistogramDump) metric;
			target.put(name + "_min", String.valueOf(histogram.min));
			target.put(name + "_max", String.valueOf(histogram.max));
			target.put(name + "_mean", String.valueOf(histogram.mean));
			target.put(name + "_median", String.valueOf(histogram.median));
			target.put(name + "_stddev", String.valueOf(histogram.stddev));
			target.put(name + "_p75", String.valueOf(histogram.p75));
			target.put(name + "_p90", String.valueOf(histogram.p90));
			target.put(name + "_p95", String.valueOf(histogram.p95));
			target.put(name + "_p98", String.valueOf(histogram.p98));
			target.put(name + "_p99", String.valueOf(histogram.p99));
			target.put(name + "_p999", String.valueOf(histogram.p999));
			break;
		case METRIC_CATEGORY_METER:
			MetricDump.MeterDump meter = (MetricDump.MeterDump) metric;
			target.put(name, String.valueOf(meter.rate));
			break;
	}
}

Example #23

Source File: MetricStore.java From flink with Apache License 2.0

5 votes

private void addMetric(Map<String, String> target, String name, MetricDump metric) {
	switch (metric.getCategory()) {
		case METRIC_CATEGORY_COUNTER:
			MetricDump.CounterDump counter = (MetricDump.CounterDump) metric;
			target.put(name, String.valueOf(counter.count));
			break;
		case METRIC_CATEGORY_GAUGE:
			MetricDump.GaugeDump gauge = (MetricDump.GaugeDump) metric;
			target.put(name, gauge.value);
			break;
		case METRIC_CATEGORY_HISTOGRAM:
			MetricDump.HistogramDump histogram = (MetricDump.HistogramDump) metric;
			target.put(name + "_min", String.valueOf(histogram.min));
			target.put(name + "_max", String.valueOf(histogram.max));
			target.put(name + "_mean", String.valueOf(histogram.mean));
			target.put(name + "_median", String.valueOf(histogram.median));
			target.put(name + "_stddev", String.valueOf(histogram.stddev));
			target.put(name + "_p75", String.valueOf(histogram.p75));
			target.put(name + "_p90", String.valueOf(histogram.p90));
			target.put(name + "_p95", String.valueOf(histogram.p95));
			target.put(name + "_p98", String.valueOf(histogram.p98));
			target.put(name + "_p99", String.valueOf(histogram.p99));
			target.put(name + "_p999", String.valueOf(histogram.p999));
			break;
		case METRIC_CATEGORY_METER:
			MetricDump.MeterDump meter = (MetricDump.MeterDump) metric;
			target.put(name, String.valueOf(meter.rate));
			break;
	}
}

Example #24

Source File: MetricStore.java From Flink-CEPplus with Apache License 2.0

5 votes

private void addMetric(Map<String, String> target, String name, MetricDump metric) {
	switch (metric.getCategory()) {
		case METRIC_CATEGORY_COUNTER:
			MetricDump.CounterDump counter = (MetricDump.CounterDump) metric;
			target.put(name, String.valueOf(counter.count));
			break;
		case METRIC_CATEGORY_GAUGE:
			MetricDump.GaugeDump gauge = (MetricDump.GaugeDump) metric;
			target.put(name, gauge.value);
			break;
		case METRIC_CATEGORY_HISTOGRAM:
			MetricDump.HistogramDump histogram = (MetricDump.HistogramDump) metric;
			target.put(name + "_min", String.valueOf(histogram.min));
			target.put(name + "_max", String.valueOf(histogram.max));
			target.put(name + "_mean", String.valueOf(histogram.mean));
			target.put(name + "_median", String.valueOf(histogram.median));
			target.put(name + "_stddev", String.valueOf(histogram.stddev));
			target.put(name + "_p75", String.valueOf(histogram.p75));
			target.put(name + "_p90", String.valueOf(histogram.p90));
			target.put(name + "_p95", String.valueOf(histogram.p95));
			target.put(name + "_p98", String.valueOf(histogram.p98));
			target.put(name + "_p99", String.valueOf(histogram.p99));
			target.put(name + "_p999", String.valueOf(histogram.p999));
			break;
		case METRIC_CATEGORY_METER:
			MetricDump.MeterDump meter = (MetricDump.MeterDump) metric;
			target.put(name, String.valueOf(meter.rate));
			break;
	}
}

Example #25

Source File: MetricStoreTest.java From flink with Apache License 2.0

4 votes

public static MetricStore setupStore(MetricStore store) {
	QueryScopeInfo.JobManagerQueryScopeInfo jm = new QueryScopeInfo.JobManagerQueryScopeInfo("abc");
	MetricDump.CounterDump cd1 = new MetricDump.CounterDump(jm, "metric1", 0);

	QueryScopeInfo.TaskManagerQueryScopeInfo tm = new QueryScopeInfo.TaskManagerQueryScopeInfo("tmid", "abc");
	MetricDump.CounterDump cd2 = new MetricDump.CounterDump(tm, "metric2", 1);
	MetricDump.CounterDump cd2a = new MetricDump.CounterDump(tm, "metric22", 1);

	QueryScopeInfo.TaskManagerQueryScopeInfo tm2 = new QueryScopeInfo.TaskManagerQueryScopeInfo("tmid2", "abc");
	MetricDump.CounterDump cd22 = new MetricDump.CounterDump(tm2, "metric2", 10);
	MetricDump.CounterDump cd22a = new MetricDump.CounterDump(tm2, "metric2b", 10);

	QueryScopeInfo.JobQueryScopeInfo job = new QueryScopeInfo.JobQueryScopeInfo("jobid", "abc");
	MetricDump.CounterDump cd3 = new MetricDump.CounterDump(job, "metric3", 2);
	MetricDump.CounterDump cd4 = new MetricDump.CounterDump(job, "metric4", 3);

	QueryScopeInfo.JobQueryScopeInfo job2 = new QueryScopeInfo.JobQueryScopeInfo("jobid2", "abc");
	MetricDump.CounterDump cd32 = new MetricDump.CounterDump(job2, "metric3", 2);
	MetricDump.CounterDump cd42 = new MetricDump.CounterDump(job2, "metric4", 3);

	QueryScopeInfo.TaskQueryScopeInfo task = new QueryScopeInfo.TaskQueryScopeInfo("jobid", "taskid", 8, "abc");
	MetricDump.CounterDump cd5 = new MetricDump.CounterDump(task, "metric5", 4);

	QueryScopeInfo.OperatorQueryScopeInfo operator = new QueryScopeInfo.OperatorQueryScopeInfo("jobid", "taskid", 8, "opname", "abc");
	MetricDump.CounterDump cd6 = new MetricDump.CounterDump(operator, "metric6", 5);
	MetricDump.CounterDump cd7 = new MetricDump.CounterDump(operator, "metric7", 6);

	QueryScopeInfo.OperatorQueryScopeInfo operator2 = new QueryScopeInfo.OperatorQueryScopeInfo("jobid", "taskid", 1, "opname", "abc");
	MetricDump.CounterDump cd62 = new MetricDump.CounterDump(operator2, "metric6", 5);
	MetricDump.CounterDump cd72 = new MetricDump.CounterDump(operator2, "metric7", 6);

	store.add(cd1);
	store.add(cd2);
	store.add(cd2a);
	store.add(cd3);
	store.add(cd4);
	store.add(cd5);
	store.add(cd6);
	store.add(cd7);

	store.add(cd62);
	store.add(cd72);
	store.add(cd22);
	store.add(cd22a);
	store.add(cd32);
	store.add(cd42);

	return store;
}

Example #26

Source File: MetricStore.java From Flink-CEPplus with Apache License 2.0

4 votes

@VisibleForTesting
public void add(MetricDump metric) {
	try {
		QueryScopeInfo info = metric.scopeInfo;
		TaskManagerMetricStore tm;
		JobMetricStore job;
		TaskMetricStore task;
		ComponentMetricStore subtask;

		String name = info.scope.isEmpty()
			? metric.name
			: info.scope + "." + metric.name;

		if (name.isEmpty()) { // malformed transmission
			return;
		}

		switch (info.getCategory()) {
			case INFO_CATEGORY_JM:
				addMetric(jobManager.metrics, name, metric);
				break;
			case INFO_CATEGORY_TM:
				String tmID = ((QueryScopeInfo.TaskManagerQueryScopeInfo) info).taskManagerID;
				tm = taskManagers.computeIfAbsent(tmID, k -> new TaskManagerMetricStore());
				if (name.contains("GarbageCollector")) {
					String gcName = name.substring("Status.JVM.GarbageCollector.".length(), name.lastIndexOf('.'));
					tm.addGarbageCollectorName(gcName);
				}
				addMetric(tm.metrics, name, metric);
				break;
			case INFO_CATEGORY_JOB:
				QueryScopeInfo.JobQueryScopeInfo jobInfo = (QueryScopeInfo.JobQueryScopeInfo) info;
				job = jobs.computeIfAbsent(jobInfo.jobID, k -> new JobMetricStore());
				addMetric(job.metrics, name, metric);
				break;
			case INFO_CATEGORY_TASK:
				QueryScopeInfo.TaskQueryScopeInfo taskInfo = (QueryScopeInfo.TaskQueryScopeInfo) info;
				job = jobs.computeIfAbsent(taskInfo.jobID, k -> new JobMetricStore());
				task = job.tasks.computeIfAbsent(taskInfo.vertexID, k -> new TaskMetricStore());
				subtask = task.subtasks.computeIfAbsent(taskInfo.subtaskIndex, k -> new ComponentMetricStore());
				/**
				 * The duplication is intended. Metrics scoped by subtask are useful for several job/task handlers,
				 * while the WebInterface task metric queries currently do not account for subtasks, so we don't
				 * divide by subtask and instead use the concatenation of subtask index and metric name as the name
				 * for those.
				 */
				addMetric(subtask.metrics, name, metric);
				addMetric(task.metrics, taskInfo.subtaskIndex + "." + name, metric);
				break;
			case INFO_CATEGORY_OPERATOR:
				QueryScopeInfo.OperatorQueryScopeInfo operatorInfo = (QueryScopeInfo.OperatorQueryScopeInfo) info;
				job = jobs.computeIfAbsent(operatorInfo.jobID, k -> new JobMetricStore());
				task = job.tasks.computeIfAbsent(operatorInfo.vertexID, k -> new TaskMetricStore());
				subtask = task.subtasks.computeIfAbsent(operatorInfo.subtaskIndex, k -> new ComponentMetricStore());
				/**
				 * As the WebInterface does not account for operators (because it can't) we don't
				 * divide by operator and instead use the concatenation of subtask index, operator name and metric name
				 * as the name.
				 */
				addMetric(subtask.metrics, operatorInfo.operatorName + "." + name, metric);
				addMetric(task.metrics, operatorInfo.subtaskIndex + "." + operatorInfo.operatorName + "." + name, metric);
				break;
			default:
				LOG.debug("Invalid metric dump category: " + info.getCategory());
		}
	} catch (Exception e) {
		LOG.debug("Malformed metric dump.", e);
	}
}

Example #27

Source File: MetricStore.java From flink with Apache License 2.0

4 votes

@VisibleForTesting
public void add(MetricDump metric) {
	try {
		QueryScopeInfo info = metric.scopeInfo;
		TaskManagerMetricStore tm;
		JobMetricStore job;
		TaskMetricStore task;
		ComponentMetricStore subtask;

		String name = info.scope.isEmpty()
			? metric.name
			: info.scope + "." + metric.name;

		if (name.isEmpty()) { // malformed transmission
			return;
		}

		switch (info.getCategory()) {
			case INFO_CATEGORY_JM:
				addMetric(jobManager.metrics, name, metric);
				break;
			case INFO_CATEGORY_TM:
				String tmID = ((QueryScopeInfo.TaskManagerQueryScopeInfo) info).taskManagerID;
				tm = taskManagers.computeIfAbsent(tmID, k -> new TaskManagerMetricStore());
				if (name.contains("GarbageCollector")) {
					String gcName = name.substring("Status.JVM.GarbageCollector.".length(), name.lastIndexOf('.'));
					tm.addGarbageCollectorName(gcName);
				}
				addMetric(tm.metrics, name, metric);
				break;
			case INFO_CATEGORY_JOB:
				QueryScopeInfo.JobQueryScopeInfo jobInfo = (QueryScopeInfo.JobQueryScopeInfo) info;
				job = jobs.computeIfAbsent(jobInfo.jobID, k -> new JobMetricStore());
				addMetric(job.metrics, name, metric);
				break;
			case INFO_CATEGORY_TASK:
				QueryScopeInfo.TaskQueryScopeInfo taskInfo = (QueryScopeInfo.TaskQueryScopeInfo) info;
				job = jobs.computeIfAbsent(taskInfo.jobID, k -> new JobMetricStore());
				task = job.tasks.computeIfAbsent(taskInfo.vertexID, k -> new TaskMetricStore());
				subtask = task.subtasks.computeIfAbsent(taskInfo.subtaskIndex, k -> new ComponentMetricStore());
				/**
				 * The duplication is intended. Metrics scoped by subtask are useful for several job/task handlers,
				 * while the WebInterface task metric queries currently do not account for subtasks, so we don't
				 * divide by subtask and instead use the concatenation of subtask index and metric name as the name
				 * for those.
				 */
				addMetric(subtask.metrics, name, metric);
				addMetric(task.metrics, taskInfo.subtaskIndex + "." + name, metric);
				break;
			case INFO_CATEGORY_OPERATOR:
				QueryScopeInfo.OperatorQueryScopeInfo operatorInfo = (QueryScopeInfo.OperatorQueryScopeInfo) info;
				job = jobs.computeIfAbsent(operatorInfo.jobID, k -> new JobMetricStore());
				task = job.tasks.computeIfAbsent(operatorInfo.vertexID, k -> new TaskMetricStore());
				subtask = task.subtasks.computeIfAbsent(operatorInfo.subtaskIndex, k -> new ComponentMetricStore());
				/**
				 * As the WebInterface does not account for operators (because it can't) we don't
				 * divide by operator and instead use the concatenation of subtask index, operator name and metric name
				 * as the name.
				 */
				addMetric(subtask.metrics, operatorInfo.operatorName + "." + name, metric);
				addMetric(task.metrics, operatorInfo.subtaskIndex + "." + operatorInfo.operatorName + "." + name, metric);
				break;
			default:
				LOG.debug("Invalid metric dump category: " + info.getCategory());
		}
	} catch (Exception e) {
		LOG.debug("Malformed metric dump.", e);
	}
}

Example #28

Source File: MetricStore.java From flink with Apache License 2.0

4 votes

/**
 * Add metric dumps to the store.
 *
 * @param metricDumps to add.
 */
synchronized void addAll(List<MetricDump> metricDumps) {
	for (MetricDump metric : metricDumps) {
		add(metric);
	}
}

Example #29

Source File: MetricStoreTest.java From flink with Apache License 2.0

4 votes

public static MetricStore setupStore(MetricStore store) {
	QueryScopeInfo.JobManagerQueryScopeInfo jm = new QueryScopeInfo.JobManagerQueryScopeInfo("abc");
	MetricDump.CounterDump cd1 = new MetricDump.CounterDump(jm, "metric1", 0);

	QueryScopeInfo.TaskManagerQueryScopeInfo tm = new QueryScopeInfo.TaskManagerQueryScopeInfo("tmid", "abc");
	MetricDump.CounterDump cd2 = new MetricDump.CounterDump(tm, "metric2", 1);
	MetricDump.CounterDump cd2a = new MetricDump.CounterDump(tm, "metric22", 1);

	QueryScopeInfo.TaskManagerQueryScopeInfo tm2 = new QueryScopeInfo.TaskManagerQueryScopeInfo("tmid2", "abc");
	MetricDump.CounterDump cd22 = new MetricDump.CounterDump(tm2, "metric2", 10);
	MetricDump.CounterDump cd22a = new MetricDump.CounterDump(tm2, "metric2b", 10);

	QueryScopeInfo.JobQueryScopeInfo job = new QueryScopeInfo.JobQueryScopeInfo("jobid", "abc");
	MetricDump.CounterDump cd3 = new MetricDump.CounterDump(job, "metric3", 2);
	MetricDump.CounterDump cd4 = new MetricDump.CounterDump(job, "metric4", 3);

	QueryScopeInfo.JobQueryScopeInfo job2 = new QueryScopeInfo.JobQueryScopeInfo("jobid2", "abc");
	MetricDump.CounterDump cd32 = new MetricDump.CounterDump(job2, "metric3", 2);
	MetricDump.CounterDump cd42 = new MetricDump.CounterDump(job2, "metric4", 3);

	QueryScopeInfo.TaskQueryScopeInfo task = new QueryScopeInfo.TaskQueryScopeInfo("jobid", "taskid", 8, "abc");
	MetricDump.CounterDump cd5 = new MetricDump.CounterDump(task, "metric5", 4);

	QueryScopeInfo.OperatorQueryScopeInfo operator = new QueryScopeInfo.OperatorQueryScopeInfo("jobid", "taskid", 8, "opname", "abc");
	MetricDump.CounterDump cd6 = new MetricDump.CounterDump(operator, "metric6", 5);
	MetricDump.CounterDump cd7 = new MetricDump.CounterDump(operator, "metric7", 6);

	QueryScopeInfo.OperatorQueryScopeInfo operator2 = new QueryScopeInfo.OperatorQueryScopeInfo("jobid", "taskid", 1, "opname", "abc");
	MetricDump.CounterDump cd62 = new MetricDump.CounterDump(operator2, "metric6", 5);
	MetricDump.CounterDump cd72 = new MetricDump.CounterDump(operator2, "metric7", 6);

	store.add(cd1);
	store.add(cd2);
	store.add(cd2a);
	store.add(cd3);
	store.add(cd4);
	store.add(cd5);
	store.add(cd6);
	store.add(cd7);

	store.add(cd62);
	store.add(cd72);
	store.add(cd22);
	store.add(cd22a);
	store.add(cd32);
	store.add(cd42);

	return store;
}

Example #30

Source File: MetricStore.java From flink with Apache License 2.0

4 votes

@VisibleForTesting
public void add(MetricDump metric) {
	try {
		QueryScopeInfo info = metric.scopeInfo;
		TaskManagerMetricStore tm;
		JobMetricStore job;
		TaskMetricStore task;
		ComponentMetricStore subtask;

		String name = info.scope.isEmpty()
			? metric.name
			: info.scope + "." + metric.name;

		if (name.isEmpty()) { // malformed transmission
			return;
		}

		switch (info.getCategory()) {
			case INFO_CATEGORY_JM:
				addMetric(jobManager.metrics, name, metric);
				break;
			case INFO_CATEGORY_TM:
				String tmID = ((QueryScopeInfo.TaskManagerQueryScopeInfo) info).taskManagerID;
				tm = taskManagers.computeIfAbsent(tmID, k -> new TaskManagerMetricStore());
				if (name.contains("GarbageCollector")) {
					String gcName = name.substring("Status.JVM.GarbageCollector.".length(), name.lastIndexOf('.'));
					tm.addGarbageCollectorName(gcName);
				}
				addMetric(tm.metrics, name, metric);
				break;
			case INFO_CATEGORY_JOB:
				QueryScopeInfo.JobQueryScopeInfo jobInfo = (QueryScopeInfo.JobQueryScopeInfo) info;
				job = jobs.computeIfAbsent(jobInfo.jobID, k -> new JobMetricStore());
				addMetric(job.metrics, name, metric);
				break;
			case INFO_CATEGORY_TASK:
				QueryScopeInfo.TaskQueryScopeInfo taskInfo = (QueryScopeInfo.TaskQueryScopeInfo) info;
				job = jobs.computeIfAbsent(taskInfo.jobID, k -> new JobMetricStore());
				task = job.tasks.computeIfAbsent(taskInfo.vertexID, k -> new TaskMetricStore());
				subtask = task.subtasks.computeIfAbsent(taskInfo.subtaskIndex, k -> new ComponentMetricStore());
				/**
				 * The duplication is intended. Metrics scoped by subtask are useful for several job/task handlers,
				 * while the WebInterface task metric queries currently do not account for subtasks, so we don't
				 * divide by subtask and instead use the concatenation of subtask index and metric name as the name
				 * for those.
				 */
				addMetric(subtask.metrics, name, metric);
				addMetric(task.metrics, taskInfo.subtaskIndex + "." + name, metric);
				break;
			case INFO_CATEGORY_OPERATOR:
				QueryScopeInfo.OperatorQueryScopeInfo operatorInfo = (QueryScopeInfo.OperatorQueryScopeInfo) info;
				job = jobs.computeIfAbsent(operatorInfo.jobID, k -> new JobMetricStore());
				task = job.tasks.computeIfAbsent(operatorInfo.vertexID, k -> new TaskMetricStore());
				subtask = task.subtasks.computeIfAbsent(operatorInfo.subtaskIndex, k -> new ComponentMetricStore());
				/**
				 * As the WebInterface does not account for operators (because it can't) we don't
				 * divide by operator and instead use the concatenation of subtask index, operator name and metric name
				 * as the name.
				 */
				addMetric(subtask.metrics, operatorInfo.operatorName + "." + name, metric);
				addMetric(task.metrics, operatorInfo.subtaskIndex + "." + operatorInfo.operatorName + "." + name, metric);
				break;
			default:
				LOG.debug("Invalid metric dump category: " + info.getCategory());
		}
	} catch (Exception e) {
		LOG.debug("Malformed metric dump.", e);
	}
}