org.apache.flink.runtime.webmonitor.retriever.MetricQueryServiceGateway Java Examples

The following examples show how to use org.apache.flink.runtime.webmonitor.retriever.MetricQueryServiceGateway. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MetricFetcherImpl.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Retrieves and queries the specified QueryServiceGateway.
 *
 * @param queryServicePath specifying the QueryServiceGateway
 */
private void retrieveAndQueryMetrics(String queryServicePath) {
	LOG.debug("Retrieve metric query service gateway for {}", queryServicePath);

	final CompletableFuture<MetricQueryServiceGateway> queryServiceGatewayFuture = queryServiceRetriever.retrieveService(queryServicePath);

	queryServiceGatewayFuture.whenCompleteAsync(
		(MetricQueryServiceGateway queryServiceGateway, Throwable t) -> {
			if (t != null) {
				LOG.debug("Could not retrieve QueryServiceGateway.", t);
			} else {
				queryMetrics(queryServiceGateway);
			}
		},
		executor);
}
Example #2
Source File: MetricRegistryImplTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testMetricQueryServiceSetup() throws Exception {
	MetricRegistryImpl metricRegistry = new MetricRegistryImpl(MetricRegistryConfiguration.defaultMetricRegistryConfiguration());

	Assert.assertNull(metricRegistry.getMetricQueryServiceGatewayRpcAddress());

	metricRegistry.startQueryService(new TestingRpcService(), new ResourceID("mqs"));

	MetricQueryServiceGateway metricQueryServiceGateway = metricRegistry.getMetricQueryServiceGateway();
	Assert.assertNotNull(metricQueryServiceGateway);

	metricRegistry.register(new SimpleCounter(), "counter", UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup());

	boolean metricsSuccessfullyQueried = false;
	for (int x = 0; x < 10; x++) {
		MetricDumpSerialization.MetricSerializationResult metricSerializationResult = metricQueryServiceGateway.queryMetrics(Time.seconds(5))
			.get(5, TimeUnit.SECONDS);

		if (metricSerializationResult.numCounters == 1) {
			metricsSuccessfullyQueried = true;
		} else {
			Thread.sleep(50);
		}
	}
	Assert.assertTrue("metrics query did not return expected result", metricsSuccessfullyQueried);
}
Example #3
Source File: MetricFetcherImpl.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Query the metrics from the given QueryServiceGateway.
 *
 * @param queryServiceGateway to query for metrics
 */
private void queryMetrics(final MetricQueryServiceGateway queryServiceGateway) {
	LOG.debug("Query metrics for {}.", queryServiceGateway.getAddress());

	queryServiceGateway
		.queryMetrics(timeout)
		.whenCompleteAsync(
			(MetricDumpSerialization.MetricSerializationResult result, Throwable t) -> {
				if (t != null) {
					LOG.debug("Fetching metrics failed.", t);
				} else {
					metrics.addAll(deserializer.deserialize(result));
				}
			},
			executor);
}
Example #4
Source File: MetricFetcherImpl.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Retrieves and queries the specified QueryServiceGateway.
 *
 * @param queryServiceAddress specifying the QueryServiceGateway
 */
private void retrieveAndQueryMetrics(String queryServiceAddress) {
	LOG.debug("Retrieve metric query service gateway for {}", queryServiceAddress);

	final CompletableFuture<MetricQueryServiceGateway> queryServiceGatewayFuture = queryServiceRetriever.retrieveService(queryServiceAddress);

	queryServiceGatewayFuture.whenCompleteAsync(
		(MetricQueryServiceGateway queryServiceGateway, Throwable t) -> {
			if (t != null) {
				LOG.debug("Could not retrieve QueryServiceGateway.", t);
			} else {
				queryMetrics(queryServiceGateway);
			}
		},
		executor);
}
Example #5
Source File: MetricRegistryImplTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testMetricQueryServiceSetup() throws Exception {
	MetricRegistryImpl metricRegistry = new MetricRegistryImpl(MetricRegistryConfiguration.defaultMetricRegistryConfiguration());

	Assert.assertNull(metricRegistry.getMetricQueryServiceGatewayRpcAddress());

	metricRegistry.startQueryService(new TestingRpcService(), new ResourceID("mqs"));

	MetricQueryServiceGateway metricQueryServiceGateway = metricRegistry.getMetricQueryServiceGateway();
	Assert.assertNotNull(metricQueryServiceGateway);

	metricRegistry.register(new SimpleCounter(), "counter", UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup());

	boolean metricsSuccessfullyQueried = false;
	for (int x = 0; x < 10; x++) {
		MetricDumpSerialization.MetricSerializationResult metricSerializationResult = metricQueryServiceGateway.queryMetrics(Time.seconds(5))
			.get(5, TimeUnit.SECONDS);

		if (metricSerializationResult.numCounters == 1) {
			metricsSuccessfullyQueried = true;
		} else {
			Thread.sleep(50);
		}
	}
	Assert.assertTrue("metrics query did not return expected result", metricsSuccessfullyQueried);
}
Example #6
Source File: MetricFetcherImpl.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Query the metrics from the given QueryServiceGateway.
 *
 * @param queryServiceGateway to query for metrics
 */
private void queryMetrics(final MetricQueryServiceGateway queryServiceGateway) {
	LOG.debug("Query metrics for {}.", queryServiceGateway.getAddress());

	queryServiceGateway
		.queryMetrics(timeout)
		.whenCompleteAsync(
			(MetricDumpSerialization.MetricSerializationResult result, Throwable t) -> {
				if (t != null) {
					LOG.debug("Fetching metrics failed.", t);
				} else {
					metrics.addAll(deserializer.deserialize(result));
				}
			},
			executor);
}
Example #7
Source File: MetricFetcherImpl.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Retrieves and queries the specified QueryServiceGateway.
 *
 * @param queryServiceAddress specifying the QueryServiceGateway
 */
private void retrieveAndQueryMetrics(String queryServiceAddress) {
	LOG.debug("Retrieve metric query service gateway for {}", queryServiceAddress);

	final CompletableFuture<MetricQueryServiceGateway> queryServiceGatewayFuture = queryServiceRetriever.retrieveService(queryServiceAddress);

	queryServiceGatewayFuture.whenCompleteAsync(
		(MetricQueryServiceGateway queryServiceGateway, Throwable t) -> {
			if (t != null) {
				LOG.debug("Could not retrieve QueryServiceGateway.", t);
			} else {
				queryMetrics(queryServiceGateway);
			}
		},
		executor);
}
Example #8
Source File: MetricFetcherImpl.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Query the metrics from the given QueryServiceGateway.
 *
 * @param queryServiceGateway to query for metrics
 */
private void queryMetrics(final MetricQueryServiceGateway queryServiceGateway) {
	LOG.debug("Query metrics for {}.", queryServiceGateway.getAddress());

	queryServiceGateway
		.queryMetrics(timeout)
		.whenCompleteAsync(
			(MetricDumpSerialization.MetricSerializationResult result, Throwable t) -> {
				if (t != null) {
					LOG.debug("Fetching metrics failed.", t);
				} else {
					metrics.addAll(deserializer.deserialize(result));
				}
			},
			executor);
}
Example #9
Source File: MetricRegistryImpl.java    From flink with Apache License 2.0 5 votes vote down vote up 
@VisibleForTesting
@Nullable
MetricQueryServiceGateway getMetricQueryServiceGateway() {
	if (queryService != null) {
		return queryService.getSelfGateway(MetricQueryServiceGateway.class);
	} else {
		return null;
	}
}
Example #10
Source File: MetricRegistryImpl.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns the address under which the {@link MetricQueryService} is reachable.
 *
 * @return address of the metric query service
 */
@Override
@Nullable
public String getMetricQueryServiceGatewayRpcAddress() {
	if (queryService != null) {
		return queryService.getSelfGateway(MetricQueryServiceGateway.class).getAddress();
	} else {
		return null;
	}
}
Example #11
Source File: MetricRegistryImpl.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns the address under which the {@link MetricQueryService} is reachable.
 *
 * @return address of the metric query service
 */
@Override
@Nullable
public String getMetricQueryServiceGatewayRpcAddress() {
	if (queryService != null) {
		return queryService.getSelfGateway(MetricQueryServiceGateway.class).getAddress();
	} else {
		return null;
	}
}
Example #12
Source File: MetricRegistryImpl.java    From flink with Apache License 2.0 5 votes vote down vote up 
@VisibleForTesting
@Nullable
MetricQueryServiceGateway getMetricQueryServiceGateway() {
	if (queryService != null) {
		return queryService.getSelfGateway(MetricQueryServiceGateway.class);
	} else {
		return null;
	}
}
Example #13
Source File: RpcMetricQueryServiceRetriever.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public CompletableFuture<MetricQueryServiceGateway> retrieveService(String rpcServiceAddress) {
	return rpcService.connect(rpcServiceAddress, MetricQueryServiceGateway.class);
}
Example #14
Source File: MetricFetcherTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testUpdate() {
	final Time timeout = Time.seconds(10L);

	// ========= setup TaskManager =================================================================================

	JobID jobID = new JobID();
	ResourceID tmRID = ResourceID.generate();

	// ========= setup QueryServices ================================================================================

	final MetricQueryServiceGateway jmQueryService = new TestingMetricQueryServiceGateway.Builder()
		.setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(new MetricDumpSerialization.MetricSerializationResult(new byte[0], new byte[0], new byte[0], new byte[0], 0, 0, 0, 0)))
		.build();

	MetricDumpSerialization.MetricSerializationResult requestMetricsAnswer = createRequestDumpAnswer(tmRID, jobID);
	final MetricQueryServiceGateway tmQueryService = new TestingMetricQueryServiceGateway.Builder()
		.setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(requestMetricsAnswer))
		.build();

	// ========= setup JobManager ==================================================================================

	final TestingRestfulGateway restfulGateway = new TestingRestfulGateway.Builder()
		.setRequestMultipleJobDetailsSupplier(() -> CompletableFuture.completedFuture(new MultipleJobsDetails(Collections.emptyList())))
		.setRequestMetricQueryServiceGatewaysSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(jmQueryService.getAddress())))
		.setRequestTaskManagerMetricQueryServiceGatewaysSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(Tuple2.of(tmRID, tmQueryService.getAddress()))))
		.build();

	final GatewayRetriever<RestfulGateway> retriever = () -> CompletableFuture.completedFuture(restfulGateway);

	// ========= start MetricFetcher testing =======================================================================
	MetricFetcher fetcher = new MetricFetcherImpl<>(
		retriever,
		address -> CompletableFuture.completedFuture(tmQueryService),
		Executors.directExecutor(),
		timeout,
		MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());

	// verify that update fetches metrics and updates the store
	fetcher.update();
	MetricStore store = fetcher.getMetricStore();
	synchronized (store) {
		assertEquals("7", store.getJobManagerMetricStore().getMetric("abc.hist_min"));
		assertEquals("6", store.getJobManagerMetricStore().getMetric("abc.hist_max"));
		assertEquals("4.0", store.getJobManagerMetricStore().getMetric("abc.hist_mean"));
		assertEquals("0.5", store.getJobManagerMetricStore().getMetric("abc.hist_median"));
		assertEquals("5.0", store.getJobManagerMetricStore().getMetric("abc.hist_stddev"));
		assertEquals("0.75", store.getJobManagerMetricStore().getMetric("abc.hist_p75"));
		assertEquals("0.9", store.getJobManagerMetricStore().getMetric("abc.hist_p90"));
		assertEquals("0.95", store.getJobManagerMetricStore().getMetric("abc.hist_p95"));
		assertEquals("0.98", store.getJobManagerMetricStore().getMetric("abc.hist_p98"));
		assertEquals("0.99", store.getJobManagerMetricStore().getMetric("abc.hist_p99"));
		assertEquals("0.999", store.getJobManagerMetricStore().getMetric("abc.hist_p999"));

		assertEquals("x", store.getTaskManagerMetricStore(tmRID.toString()).metrics.get("abc.gauge"));
		assertEquals("5.0", store.getJobMetricStore(jobID.toString()).metrics.get("abc.jc"));
		assertEquals("2", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.abc.tc"));
		assertEquals("1", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.opname.abc.oc"));
	}
}
Example #15
Source File: RpcMetricQueryServiceRetriever.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public CompletableFuture<MetricQueryServiceGateway> retrieveService(String rpcServiceAddress) {
	return rpcService.connect(rpcServiceAddress, MetricQueryServiceGateway.class);
}
Example #16
Source File: MetricFetcherTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Test
public void testUpdate() {
	final Time timeout = Time.seconds(10L);

	// ========= setup TaskManager =================================================================================
	JobID jobID = new JobID();
	ResourceID tmRID = ResourceID.generate();

	// ========= setup JobManager ==================================================================================

	final String jmMetricQueryServicePath = "/jm/" + MetricQueryService.METRIC_QUERY_SERVICE_NAME;
	final String tmMetricQueryServicePath = "/tm/" + MetricQueryService.METRIC_QUERY_SERVICE_NAME + "_" + tmRID.getResourceIdString();

	final TestingRestfulGateway restfulGateway = new TestingRestfulGateway.Builder()
		.setRequestMultipleJobDetailsSupplier(() -> CompletableFuture.completedFuture(new MultipleJobsDetails(Collections.emptyList())))
		.setRequestMetricQueryServicePathsSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(jmMetricQueryServicePath)))
		.setRequestTaskManagerMetricQueryServicePathsSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(Tuple2.of(tmRID, tmMetricQueryServicePath))))
		.build();

	final GatewayRetriever<RestfulGateway> retriever = () -> CompletableFuture.completedFuture(restfulGateway);

	// ========= setup QueryServices ================================================================================
	final MetricQueryServiceGateway jmQueryService = new TestingMetricQueryServiceGateway.Builder()
		.setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(new MetricDumpSerialization.MetricSerializationResult(new byte[0], new byte[0], new byte[0], new byte[0], 0, 0, 0, 0)))
		.build();

	MetricDumpSerialization.MetricSerializationResult requestMetricsAnswer = createRequestDumpAnswer(tmRID, jobID);
	final MetricQueryServiceGateway tmQueryService = new TestingMetricQueryServiceGateway.Builder()
		.setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(requestMetricsAnswer))
		.build();

	final MetricQueryServiceRetriever queryServiceRetriever = (path) -> {
		if (path.equals(jmMetricQueryServicePath))  {
			return CompletableFuture.completedFuture(jmQueryService);
		} else if (path.equals(tmMetricQueryServicePath)) {
			return CompletableFuture.completedFuture(tmQueryService);
		} else {
			throw new IllegalArgumentException("Unexpected argument.");
		}
	};

	// ========= start MetricFetcher testing =======================================================================
	MetricFetcher fetcher = new MetricFetcherImpl<>(
		retriever,
		queryServiceRetriever,
		Executors.directExecutor(),
		timeout,
		MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());

	// verify that update fetches metrics and updates the store
	fetcher.update();
	MetricStore store = fetcher.getMetricStore();
	synchronized (store) {
		assertEquals("7", store.getJobManagerMetricStore().getMetric("abc.hist_min"));
		assertEquals("6", store.getJobManagerMetricStore().getMetric("abc.hist_max"));
		assertEquals("4.0", store.getJobManagerMetricStore().getMetric("abc.hist_mean"));
		assertEquals("0.5", store.getJobManagerMetricStore().getMetric("abc.hist_median"));
		assertEquals("5.0", store.getJobManagerMetricStore().getMetric("abc.hist_stddev"));
		assertEquals("0.75", store.getJobManagerMetricStore().getMetric("abc.hist_p75"));
		assertEquals("0.9", store.getJobManagerMetricStore().getMetric("abc.hist_p90"));
		assertEquals("0.95", store.getJobManagerMetricStore().getMetric("abc.hist_p95"));
		assertEquals("0.98", store.getJobManagerMetricStore().getMetric("abc.hist_p98"));
		assertEquals("0.99", store.getJobManagerMetricStore().getMetric("abc.hist_p99"));
		assertEquals("0.999", store.getJobManagerMetricStore().getMetric("abc.hist_p999"));

		assertEquals("x", store.getTaskManagerMetricStore(tmRID.toString()).metrics.get("abc.gauge"));
		assertEquals("5.0", store.getJobMetricStore(jobID.toString()).metrics.get("abc.jc"));
		assertEquals("2", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.abc.tc"));
		assertEquals("1", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.opname.abc.oc"));
	}
}
Example #17
Source File: AkkaQueryServiceRetriever.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public CompletableFuture<MetricQueryServiceGateway> retrieveService(String queryServicePath) {
	ActorSelection selection = actorSystem.actorSelection(queryServicePath);

	return FutureUtils.toJava(selection.resolveOne(FutureUtils.toFiniteDuration(lookupTimeout))).thenApply(AkkaQueryServiceGateway::new);
}
Example #18
Source File: MetricFetcherTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testUpdate() {
	final Time timeout = Time.seconds(10L);

	// ========= setup TaskManager =================================================================================

	JobID jobID = new JobID();
	ResourceID tmRID = ResourceID.generate();

	// ========= setup QueryServices ================================================================================

	final MetricQueryServiceGateway jmQueryService = new TestingMetricQueryServiceGateway.Builder()
		.setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(new MetricDumpSerialization.MetricSerializationResult(new byte[0], new byte[0], new byte[0], new byte[0], 0, 0, 0, 0)))
		.build();

	MetricDumpSerialization.MetricSerializationResult requestMetricsAnswer = createRequestDumpAnswer(tmRID, jobID);
	final MetricQueryServiceGateway tmQueryService = new TestingMetricQueryServiceGateway.Builder()
		.setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(requestMetricsAnswer))
		.build();

	// ========= setup JobManager ==================================================================================

	final TestingRestfulGateway restfulGateway = new TestingRestfulGateway.Builder()
		.setRequestMultipleJobDetailsSupplier(() -> CompletableFuture.completedFuture(new MultipleJobsDetails(Collections.emptyList())))
		.setRequestMetricQueryServiceGatewaysSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(jmQueryService.getAddress())))
		.setRequestTaskManagerMetricQueryServiceGatewaysSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(Tuple2.of(tmRID, tmQueryService.getAddress()))))
		.build();

	final GatewayRetriever<RestfulGateway> retriever = () -> CompletableFuture.completedFuture(restfulGateway);

	// ========= start MetricFetcher testing =======================================================================
	MetricFetcher fetcher = new MetricFetcherImpl<>(
		retriever,
		address -> CompletableFuture.completedFuture(tmQueryService),
		Executors.directExecutor(),
		timeout,
		MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());

	// verify that update fetches metrics and updates the store
	fetcher.update();
	MetricStore store = fetcher.getMetricStore();
	synchronized (store) {
		assertEquals("7", store.getJobManagerMetricStore().getMetric("abc.hist_min"));
		assertEquals("6", store.getJobManagerMetricStore().getMetric("abc.hist_max"));
		assertEquals("4.0", store.getJobManagerMetricStore().getMetric("abc.hist_mean"));
		assertEquals("0.5", store.getJobManagerMetricStore().getMetric("abc.hist_median"));
		assertEquals("5.0", store.getJobManagerMetricStore().getMetric("abc.hist_stddev"));
		assertEquals("0.75", store.getJobManagerMetricStore().getMetric("abc.hist_p75"));
		assertEquals("0.9", store.getJobManagerMetricStore().getMetric("abc.hist_p90"));
		assertEquals("0.95", store.getJobManagerMetricStore().getMetric("abc.hist_p95"));
		assertEquals("0.98", store.getJobManagerMetricStore().getMetric("abc.hist_p98"));
		assertEquals("0.99", store.getJobManagerMetricStore().getMetric("abc.hist_p99"));
		assertEquals("0.999", store.getJobManagerMetricStore().getMetric("abc.hist_p999"));

		assertEquals("x", store.getTaskManagerMetricStore(tmRID.toString()).metrics.get("abc.gauge"));
		assertEquals("5.0", store.getJobMetricStore(jobID.toString()).metrics.get("abc.jc"));
		assertEquals("2", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.abc.tc"));
		assertEquals("1", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.opname.abc.oc"));
	}
}