org.apache.flink.runtime.query.KvStateLocation Java Examples

The following examples show how to use org.apache.flink.runtime.query.KvStateLocation. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: KvStateClientProxyHandler.java From flink with Apache License 2.0

6 votes

private CompletableFuture<KvStateResponse> getState(
		final KvStateRequest request,
		final boolean forceUpdate) {

	return getKvStateLookupInfo(request.getJobId(), request.getStateName(), forceUpdate)
			.thenComposeAsync((Function<KvStateLocation, CompletableFuture<KvStateResponse>>) location -> {
				final int keyGroupIndex = KeyGroupRangeAssignment.computeKeyGroupForKeyHash(
						request.getKeyHashCode(), location.getNumKeyGroups());

				final InetSocketAddress serverAddress = location.getKvStateServerAddress(keyGroupIndex);
				if (serverAddress == null) {
					return FutureUtils.completedExceptionally(new UnknownKvStateKeyGroupLocationException(getServerName()));
				} else {
					// Query server
					final KvStateID kvStateId = location.getKvStateID(keyGroupIndex);
					final KvStateInternalRequest internalRequest = new KvStateInternalRequest(
							kvStateId, request.getSerializedKeyAndNamespace());
					return kvStateClient.sendRequest(serverAddress, internalRequest);
				}
			}, queryExecutor);
}

Example #2

Source File: JobMaster.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public CompletableFuture<KvStateLocation> requestKvStateLocation(final JobID jobId, final String registrationName) {
	// sanity check for the correct JobID
	if (jobGraph.getJobID().equals(jobId)) {
		if (log.isDebugEnabled()) {
			log.debug("Lookup key-value state for job {} with registration " +
				"name {}.", jobGraph.getJobID(), registrationName);
		}

		final KvStateLocationRegistry registry = executionGraph.getKvStateLocationRegistry();
		final KvStateLocation location = registry.getKvStateLocation(registrationName);
		if (location != null) {
			return CompletableFuture.completedFuture(location);
		} else {
			return FutureUtils.completedExceptionally(new UnknownKvStateLocation(registrationName));
		}
	} else {
		if (log.isDebugEnabled()) {
			log.debug("Request of key-value state location for unknown job {} received.", jobId);
		}
		return FutureUtils.completedExceptionally(new FlinkJobNotFoundException(jobId));
	}
}

Example #3

Source File: KvStateClientProxyHandler.java From flink with Apache License 2.0

6 votes

private CompletableFuture<KvStateResponse> getState(
		final KvStateRequest request,
		final boolean forceUpdate) {

	return getKvStateLookupInfo(request.getJobId(), request.getStateName(), forceUpdate)
			.thenComposeAsync((Function<KvStateLocation, CompletableFuture<KvStateResponse>>) location -> {
				final int keyGroupIndex = KeyGroupRangeAssignment.computeKeyGroupForKeyHash(
						request.getKeyHashCode(), location.getNumKeyGroups());

				final InetSocketAddress serverAddress = location.getKvStateServerAddress(keyGroupIndex);
				if (serverAddress == null) {
					return FutureUtils.completedExceptionally(new UnknownKvStateKeyGroupLocationException(getServerName()));
				} else {
					// Query server
					final KvStateID kvStateId = location.getKvStateID(keyGroupIndex);
					final KvStateInternalRequest internalRequest = new KvStateInternalRequest(
							kvStateId, request.getSerializedKeyAndNamespace());
					return kvStateClient.sendRequest(serverAddress, internalRequest);
				}
			}, queryExecutor);
}

Example #4

Source File: KvStateClientProxyHandler.java From Flink-CEPplus with Apache License 2.0

6 votes

private CompletableFuture<KvStateResponse> getState(
		final KvStateRequest request,
		final boolean forceUpdate) {

	return getKvStateLookupInfo(request.getJobId(), request.getStateName(), forceUpdate)
			.thenComposeAsync((Function<KvStateLocation, CompletableFuture<KvStateResponse>>) location -> {
				final int keyGroupIndex = KeyGroupRangeAssignment.computeKeyGroupForKeyHash(
						request.getKeyHashCode(), location.getNumKeyGroups());

				final InetSocketAddress serverAddress = location.getKvStateServerAddress(keyGroupIndex);
				if (serverAddress == null) {
					return FutureUtils.completedExceptionally(new UnknownKvStateKeyGroupLocationException(getServerName()));
				} else {
					// Query server
					final KvStateID kvStateId = location.getKvStateID(keyGroupIndex);
					final KvStateInternalRequest internalRequest = new KvStateInternalRequest(
							kvStateId, request.getSerializedKeyAndNamespace());
					return kvStateClient.sendRequest(serverAddress, internalRequest);
				}
			}, queryExecutor);
}

Example #5

Source File: LegacyScheduler.java From flink with Apache License 2.0

6 votes

@Override
public KvStateLocation requestKvStateLocation(final JobID jobId, final String registrationName) throws UnknownKvStateLocation, FlinkJobNotFoundException {
	mainThreadExecutor.assertRunningInMainThread();

	// sanity check for the correct JobID
	if (jobGraph.getJobID().equals(jobId)) {
		if (log.isDebugEnabled()) {
			log.debug("Lookup key-value state for job {} with registration " +
				"name {}.", jobGraph.getJobID(), registrationName);
		}

		final KvStateLocationRegistry registry = executionGraph.getKvStateLocationRegistry();
		final KvStateLocation location = registry.getKvStateLocation(registrationName);
		if (location != null) {
			return location;
		} else {
			throw new UnknownKvStateLocation(registrationName);
		}
	} else {
		if (log.isDebugEnabled()) {
			log.debug("Request of key-value state location for unknown job {} received.", jobId);
		}
		throw new FlinkJobNotFoundException(jobId);
	}
}

Example #6

Source File: SchedulerBase.java From flink with Apache License 2.0

6 votes

@Override
public KvStateLocation requestKvStateLocation(final JobID jobId, final String registrationName) throws UnknownKvStateLocation, FlinkJobNotFoundException {
	mainThreadExecutor.assertRunningInMainThread();

	// sanity check for the correct JobID
	if (jobGraph.getJobID().equals(jobId)) {
		if (log.isDebugEnabled()) {
			log.debug("Lookup key-value state for job {} with registration " +
				"name {}.", jobGraph.getJobID(), registrationName);
		}

		final KvStateLocationRegistry registry = executionGraph.getKvStateLocationRegistry();
		final KvStateLocation location = registry.getKvStateLocation(registrationName);
		if (location != null) {
			return location;
		} else {
			throw new UnknownKvStateLocation(registrationName);
		}
	} else {
		if (log.isDebugEnabled()) {
			log.debug("Request of key-value state location for unknown job {} received.", jobId);
		}
		throw new FlinkJobNotFoundException(jobId);
	}
}

Example #7

Source File: ActorGatewayKvStateLocationOracle.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public CompletableFuture<KvStateLocation> requestKvStateLocation(JobID jobId, String registrationName) {
	final KvStateMessage.LookupKvStateLocation lookupKvStateLocation = new KvStateMessage.LookupKvStateLocation(jobId, registrationName);

	return FutureUtils.toJava(
		jobManagerActorGateway
			.ask(lookupKvStateLocation, timeout)
			.mapTo(ClassTag$.MODULE$.<KvStateLocation>apply(KvStateLocation.class)));
}

Example #8

Source File: JobMaster.java From flink with Apache License 2.0

5 votes

@Override
public CompletableFuture<KvStateLocation> requestKvStateLocation(final JobID jobId, final String registrationName) {
	try {
		return CompletableFuture.completedFuture(schedulerNG.requestKvStateLocation(jobId, registrationName));
	} catch (UnknownKvStateLocation | FlinkJobNotFoundException e) {
		log.info("Error while request key-value state location", e);
		return FutureUtils.completedExceptionally(e);
	}
}

Example #9

Source File: JobMaster.java From flink with Apache License 2.0

5 votes

@Override
public CompletableFuture<KvStateLocation> requestKvStateLocation(final JobID jobId, final String registrationName) {
	try {
		return CompletableFuture.completedFuture(schedulerNG.requestKvStateLocation(jobId, registrationName));
	} catch (UnknownKvStateLocation | FlinkJobNotFoundException e) {
		log.info("Error while request key-value state location", e);
		return FutureUtils.completedExceptionally(e);
	}
}

Example #10

Source File: TestingJobMasterGatewayBuilder.java From flink with Apache License 2.0

4 votes

public TestingJobMasterGatewayBuilder setRequestKvStateLocationFunction(BiFunction<JobID, String, CompletableFuture<KvStateLocation>> requestKvStateLocationFunction) {
	this.requestKvStateLocationFunction = requestKvStateLocationFunction;
	return this;
}

Example #11

Source File: KvStateClientProxyHandler.java From flink with Apache License 2.0

4 votes

/**
 * Lookup the {@link KvStateLocation} for the given job and queryable state name.
 *
 * <p>The job manager will be queried for the location only if forced or no
 * cached location can be found. There are no guarantees about
 *
 * @param jobId              JobID the state instance belongs to.
 * @param queryableStateName Name under which the state instance has been published.
 * @param forceUpdate        Flag to indicate whether to force a update via the lookup service.
 * @return Future holding the KvStateLocation
 */
private CompletableFuture<KvStateLocation> getKvStateLookupInfo(
		final JobID jobId,
		final String queryableStateName,
		final boolean forceUpdate) {

	final Tuple2<JobID, String> cacheKey = new Tuple2<>(jobId, queryableStateName);
	final CompletableFuture<KvStateLocation> cachedFuture = lookupCache.get(cacheKey);

	if (!forceUpdate && cachedFuture != null && !cachedFuture.isCompletedExceptionally()) {
		LOG.debug("Retrieving location for state={} of job={} from the cache.", queryableStateName, jobId);
		return cachedFuture;
	}

	final KvStateLocationOracle kvStateLocationOracle = proxy.getKvStateLocationOracle(jobId);

	if (kvStateLocationOracle != null) {
		LOG.debug("Retrieving location for state={} of job={} from the key-value state location oracle.", queryableStateName, jobId);
		final CompletableFuture<KvStateLocation> location = new CompletableFuture<>();
		lookupCache.put(cacheKey, location);

		kvStateLocationOracle
			.requestKvStateLocation(jobId, queryableStateName)
			.whenComplete(
				(KvStateLocation kvStateLocation, Throwable throwable) -> {
					if (throwable != null) {
						if (ExceptionUtils.stripCompletionException(throwable) instanceof FlinkJobNotFoundException) {
							// if the jobId was wrong, remove the entry from the cache.
							lookupCache.remove(cacheKey);
						}
						location.completeExceptionally(throwable);
					} else {
						location.complete(kvStateLocation);
					}
				});

		return location;
	} else {
		return FutureUtils.completedExceptionally(
			new UnknownLocationException(
					"Could not retrieve location of state=" + queryableStateName + " of job=" + jobId +
							". Potential reasons are: i) the state is not ready, or ii) the job does not exist."));
	}
}

Example #12

Source File: KvStateClientProxyImplTest.java From flink with Apache License 2.0

4 votes

@Override
public CompletableFuture<KvStateLocation> requestKvStateLocation(JobID jobId, String registrationName) {
	return null;
}

Example #13

Source File: TestingJobMasterGateway.java From flink with Apache License 2.0

4 votes

@Override
public CompletableFuture<KvStateLocation> requestKvStateLocation(JobID jobId, String registrationName) {
	return requestKvStateLocationFunction.apply(jobId, registrationName);
}

Example #14

Source File: TestingJobMasterGateway.java From flink with Apache License 2.0

4 votes

public TestingJobMasterGateway(
		@Nonnull String address,
		@Nonnull String hostname,
		@Nonnull Supplier<CompletableFuture<Acknowledge>> cancelFunction,
		@Nonnull Function<TaskExecutionState, CompletableFuture<Acknowledge>> updateTaskExecutionStateFunction,
		@Nonnull BiFunction<JobVertexID, ExecutionAttemptID, CompletableFuture<SerializedInputSplit>> requestNextInputSplitFunction,
		@Nonnull BiFunction<IntermediateDataSetID, ResultPartitionID, CompletableFuture<ExecutionState>> requestPartitionStateFunction,
		@Nonnull Function<ResultPartitionID, CompletableFuture<Acknowledge>> scheduleOrUpdateConsumersFunction,
		@Nonnull Function<ResourceID, CompletableFuture<Acknowledge>> disconnectTaskManagerFunction,
		@Nonnull Consumer<ResourceManagerId> disconnectResourceManagerConsumer,
		@Nonnull BiFunction<ResourceID, Collection<SlotOffer>, CompletableFuture<Collection<SlotOffer>>> offerSlotsFunction,
		@Nonnull TriConsumer<ResourceID, AllocationID, Throwable> failSlotConsumer,
		@Nonnull BiFunction<String, UnresolvedTaskManagerLocation, CompletableFuture<RegistrationResponse>> registerTaskManagerFunction,
		@Nonnull BiConsumer<ResourceID, AccumulatorReport> taskManagerHeartbeatConsumer,
		@Nonnull Consumer<ResourceID> resourceManagerHeartbeatConsumer,
		@Nonnull Supplier<CompletableFuture<JobDetails>> requestJobDetailsSupplier,
		@Nonnull Supplier<CompletableFuture<ArchivedExecutionGraph>> requestJobSupplier,
		@Nonnull BiFunction<String, Boolean, CompletableFuture<String>> triggerSavepointFunction,
		@Nonnull BiFunction<String, Boolean, CompletableFuture<String>> stopWithSavepointFunction,
		@Nonnull Function<JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOperatorBackPressureStatsFunction,
		@Nonnull BiConsumer<AllocationID, Throwable> notifyAllocationFailureConsumer,
		@Nonnull Consumer<Tuple5<JobID, ExecutionAttemptID, Long, CheckpointMetrics, TaskStateSnapshot>> acknowledgeCheckpointConsumer,
		@Nonnull Consumer<DeclineCheckpoint> declineCheckpointConsumer,
		@Nonnull Supplier<JobMasterId> fencingTokenSupplier,
		@Nonnull BiFunction<JobID, String, CompletableFuture<KvStateLocation>> requestKvStateLocationFunction,
		@Nonnull Function<Tuple6<JobID, JobVertexID, KeyGroupRange, String, KvStateID, InetSocketAddress>, CompletableFuture<Acknowledge>> notifyKvStateRegisteredFunction,
		@Nonnull Function<Tuple4<JobID, JobVertexID, KeyGroupRange, String>, CompletableFuture<Acknowledge>> notifyKvStateUnregisteredFunction,
		@Nonnull TriFunction<String, Object, byte[], CompletableFuture<Object>> updateAggregateFunction,
		@Nonnull TriFunction<ExecutionAttemptID, OperatorID, SerializedValue<OperatorEvent>, CompletableFuture<Acknowledge>> operatorEventSender,
		@Nonnull BiFunction<OperatorID, SerializedValue<CoordinationRequest>, CompletableFuture<CoordinationResponse>> deliverCoordinationRequestFunction) {
	this.address = address;
	this.hostname = hostname;
	this.cancelFunction = cancelFunction;
	this.updateTaskExecutionStateFunction = updateTaskExecutionStateFunction;
	this.requestNextInputSplitFunction = requestNextInputSplitFunction;
	this.requestPartitionStateFunction = requestPartitionStateFunction;
	this.scheduleOrUpdateConsumersFunction = scheduleOrUpdateConsumersFunction;
	this.disconnectTaskManagerFunction = disconnectTaskManagerFunction;
	this.disconnectResourceManagerConsumer = disconnectResourceManagerConsumer;
	this.offerSlotsFunction = offerSlotsFunction;
	this.failSlotConsumer = failSlotConsumer;
	this.registerTaskManagerFunction = registerTaskManagerFunction;
	this.taskManagerHeartbeatConsumer = taskManagerHeartbeatConsumer;
	this.resourceManagerHeartbeatConsumer = resourceManagerHeartbeatConsumer;
	this.requestJobDetailsSupplier = requestJobDetailsSupplier;
	this.requestJobSupplier = requestJobSupplier;
	this.triggerSavepointFunction = triggerSavepointFunction;
	this.stopWithSavepointFunction = stopWithSavepointFunction;
	this.requestOperatorBackPressureStatsFunction = requestOperatorBackPressureStatsFunction;
	this.notifyAllocationFailureConsumer = notifyAllocationFailureConsumer;
	this.acknowledgeCheckpointConsumer = acknowledgeCheckpointConsumer;
	this.declineCheckpointConsumer = declineCheckpointConsumer;
	this.fencingTokenSupplier = fencingTokenSupplier;
	this.requestKvStateLocationFunction = requestKvStateLocationFunction;
	this.notifyKvStateRegisteredFunction = notifyKvStateRegisteredFunction;
	this.notifyKvStateUnregisteredFunction = notifyKvStateUnregisteredFunction;
	this.updateAggregateFunction = updateAggregateFunction;
	this.operatorEventSender = operatorEventSender;
	this.deliverCoordinationRequestFunction = deliverCoordinationRequestFunction;
}

Example #15

Source File: TestingJobMasterGateway.java From flink with Apache License 2.0

4 votes

@Override
public CompletableFuture<KvStateLocation> requestKvStateLocation(JobID jobId, String registrationName) {
	return requestKvStateLocationFunction.apply(jobId, registrationName);
}

Example #16

Source File: JobMasterTest.java From flink with Apache License 2.0

4 votes

@Test
public void testRegisterAndUnregisterKvState() throws Exception {
	final JobGraph graph = createKvJobGraph();
	final List<JobVertex> jobVertices = graph.getVerticesSortedTopologicallyFromSources();
	final JobVertex vertex1 = jobVertices.get(0);

	final JobMaster jobMaster = createJobMaster(
		configuration,
		graph,
		haServices,
		new TestingJobManagerSharedServicesBuilder().build(),
		heartbeatServices);

	CompletableFuture<Acknowledge> startFuture = jobMaster.start(jobMasterId);
	final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);

	try {
		// wait for the start to complete
		startFuture.get(testingTimeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		// register a KvState
		final String registrationName = "register-me";
		final KvStateID kvStateID = new KvStateID();
		final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
		final InetSocketAddress address = new InetSocketAddress(InetAddress.getLocalHost(), 1029);

		jobMasterGateway.notifyKvStateRegistered(
			graph.getJobID(),
			vertex1.getID(),
			keyGroupRange,
			registrationName,
			kvStateID,
			address).get();

		final KvStateLocation location = jobMasterGateway.requestKvStateLocation(graph.getJobID(), registrationName).get();

		assertEquals(graph.getJobID(), location.getJobId());
		assertEquals(vertex1.getID(), location.getJobVertexId());
		assertEquals(vertex1.getMaxParallelism(), location.getNumKeyGroups());
		assertEquals(1, location.getNumRegisteredKeyGroups());
		assertEquals(1, keyGroupRange.getNumberOfKeyGroups());
		assertEquals(kvStateID, location.getKvStateID(keyGroupRange.getStartKeyGroup()));
		assertEquals(address, location.getKvStateServerAddress(keyGroupRange.getStartKeyGroup()));

		// unregister the KvState
		jobMasterGateway.notifyKvStateUnregistered(
			graph.getJobID(),
			vertex1.getID(),
			keyGroupRange,
			registrationName).get();

		try {
			jobMasterGateway.requestKvStateLocation(graph.getJobID(), registrationName).get();
			fail("Expected to fail with an UnknownKvStateLocation.");
		} catch (Exception e) {
			assertTrue(ExceptionUtils.findThrowable(e, UnknownKvStateLocation.class).isPresent());
		}
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
	}
}

Example #17

Source File: KvStateClientProxyHandler.java From flink with Apache License 2.0

4 votes

/**
 * Lookup the {@link KvStateLocation} for the given job and queryable state name.
 *
 * <p>The job manager will be queried for the location only if forced or no
 * cached location can be found. There are no guarantees about
 *
 * @param jobId              JobID the state instance belongs to.
 * @param queryableStateName Name under which the state instance has been published.
 * @param forceUpdate        Flag to indicate whether to force a update via the lookup service.
 * @return Future holding the KvStateLocation
 */
private CompletableFuture<KvStateLocation> getKvStateLookupInfo(
		final JobID jobId,
		final String queryableStateName,
		final boolean forceUpdate) {

	final Tuple2<JobID, String> cacheKey = new Tuple2<>(jobId, queryableStateName);
	final CompletableFuture<KvStateLocation> cachedFuture = lookupCache.get(cacheKey);

	if (!forceUpdate && cachedFuture != null && !cachedFuture.isCompletedExceptionally()) {
		LOG.debug("Retrieving location for state={} of job={} from the cache.", queryableStateName, jobId);
		return cachedFuture;
	}

	final KvStateLocationOracle kvStateLocationOracle = proxy.getKvStateLocationOracle(jobId);

	if (kvStateLocationOracle != null) {
		LOG.debug("Retrieving location for state={} of job={} from the key-value state location oracle.", queryableStateName, jobId);
		final CompletableFuture<KvStateLocation> location = new CompletableFuture<>();
		lookupCache.put(cacheKey, location);

		kvStateLocationOracle
			.requestKvStateLocation(jobId, queryableStateName)
			.whenComplete(
				(KvStateLocation kvStateLocation, Throwable throwable) -> {
					if (throwable != null) {
						if (ExceptionUtils.stripCompletionException(throwable) instanceof FlinkJobNotFoundException) {
							// if the jobId was wrong, remove the entry from the cache.
							lookupCache.remove(cacheKey);
						}
						location.completeExceptionally(throwable);
					} else {
						location.complete(kvStateLocation);
					}
				});

		return location;
	} else {
		return FutureUtils.completedExceptionally(
			new UnknownLocationException(
					"Could not retrieve location of state=" + queryableStateName + " of job=" + jobId +
							". Potential reasons are: i) the state is not ready, or ii) the job does not exist."));
	}
}

Example #18

Source File: KvStateClientProxyImplTest.java From flink with Apache License 2.0

4 votes

@Override
public CompletableFuture<KvStateLocation> requestKvStateLocation(JobID jobId, String registrationName) {
	return null;
}

Example #19

Source File: JobMasterTest.java From flink with Apache License 2.0

4 votes

@Test
public void testRegisterAndUnregisterKvState() throws Exception {
	final JobGraph graph = createKvJobGraph();
	final List<JobVertex> jobVertices = graph.getVerticesSortedTopologicallyFromSources();
	final JobVertex vertex1 = jobVertices.get(0);

	final JobMaster jobMaster = createJobMaster(
		configuration,
		graph,
		haServices,
		new TestingJobManagerSharedServicesBuilder().build(),
		heartbeatServices);

	CompletableFuture<Acknowledge> startFuture = jobMaster.start(jobMasterId);
	final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);

	try {
		// wait for the start to complete
		startFuture.get(testingTimeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		// register a KvState
		final String registrationName = "register-me";
		final KvStateID kvStateID = new KvStateID();
		final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
		final InetSocketAddress address = new InetSocketAddress(InetAddress.getLocalHost(), 1029);

		jobMasterGateway.notifyKvStateRegistered(
			graph.getJobID(),
			vertex1.getID(),
			keyGroupRange,
			registrationName,
			kvStateID,
			address).get();

		final KvStateLocation location = jobMasterGateway.requestKvStateLocation(graph.getJobID(), registrationName).get();

		assertEquals(graph.getJobID(), location.getJobId());
		assertEquals(vertex1.getID(), location.getJobVertexId());
		assertEquals(vertex1.getMaxParallelism(), location.getNumKeyGroups());
		assertEquals(1, location.getNumRegisteredKeyGroups());
		assertEquals(1, keyGroupRange.getNumberOfKeyGroups());
		assertEquals(kvStateID, location.getKvStateID(keyGroupRange.getStartKeyGroup()));
		assertEquals(address, location.getKvStateServerAddress(keyGroupRange.getStartKeyGroup()));

		// unregister the KvState
		jobMasterGateway.notifyKvStateUnregistered(
			graph.getJobID(),
			vertex1.getID(),
			keyGroupRange,
			registrationName).get();

		try {
			jobMasterGateway.requestKvStateLocation(graph.getJobID(), registrationName).get();
			fail("Expected to fail with an UnknownKvStateLocation.");
		} catch (Exception e) {
			assertTrue(ExceptionUtils.findThrowable(e, UnknownKvStateLocation.class).isPresent());
		}
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
	}
}

Example #20

Source File: TestingJobMasterGateway.java From flink with Apache License 2.0

4 votes

public TestingJobMasterGateway(
		@Nonnull String address,
		@Nonnull String hostname,
		@Nonnull Supplier<CompletableFuture<Acknowledge>> cancelFunction,
		@Nonnull Function<TaskExecutionState, CompletableFuture<Acknowledge>> updateTaskExecutionStateFunction,
		@Nonnull BiFunction<JobVertexID, ExecutionAttemptID, CompletableFuture<SerializedInputSplit>> requestNextInputSplitFunction,
		@Nonnull BiFunction<IntermediateDataSetID, ResultPartitionID, CompletableFuture<ExecutionState>> requestPartitionStateFunction,
		@Nonnull Function<ResultPartitionID, CompletableFuture<Acknowledge>> scheduleOrUpdateConsumersFunction,
		@Nonnull Function<ResourceID, CompletableFuture<Acknowledge>> disconnectTaskManagerFunction,
		@Nonnull Consumer<ResourceManagerId> disconnectResourceManagerConsumer,
		@Nonnull Supplier<CompletableFuture<ClassloadingProps>> classloadingPropsSupplier,
		@Nonnull BiFunction<ResourceID, Collection<SlotOffer>, CompletableFuture<Collection<SlotOffer>>> offerSlotsFunction,
		@Nonnull TriConsumer<ResourceID, AllocationID, Throwable> failSlotConsumer,
		@Nonnull BiFunction<String, TaskManagerLocation, CompletableFuture<RegistrationResponse>> registerTaskManagerFunction,
		@Nonnull BiConsumer<ResourceID, AccumulatorReport> taskManagerHeartbeatConsumer,
		@Nonnull Consumer<ResourceID> resourceManagerHeartbeatConsumer,
		@Nonnull Supplier<CompletableFuture<JobDetails>> requestJobDetailsSupplier,
		@Nonnull Supplier<CompletableFuture<ArchivedExecutionGraph>> requestJobSupplier,
		@Nonnull BiFunction<String, Boolean, CompletableFuture<String>> triggerSavepointFunction,
		@Nonnull BiFunction<String, Boolean, CompletableFuture<String>> stopWithSavepointFunction,
		@Nonnull Function<JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOperatorBackPressureStatsFunction,
		@Nonnull BiConsumer<AllocationID, Throwable> notifyAllocationFailureConsumer,
		@Nonnull Consumer<Tuple5<JobID, ExecutionAttemptID, Long, CheckpointMetrics, TaskStateSnapshot>> acknowledgeCheckpointConsumer,
		@Nonnull Consumer<DeclineCheckpoint> declineCheckpointConsumer,
		@Nonnull Supplier<JobMasterId> fencingTokenSupplier,
		@Nonnull BiFunction<JobID, String, CompletableFuture<KvStateLocation>> requestKvStateLocationFunction,
		@Nonnull Function<Tuple6<JobID, JobVertexID, KeyGroupRange, String, KvStateID, InetSocketAddress>, CompletableFuture<Acknowledge>> notifyKvStateRegisteredFunction,
		@Nonnull Function<Tuple4<JobID, JobVertexID, KeyGroupRange, String>, CompletableFuture<Acknowledge>> notifyKvStateUnregisteredFunction,
		@Nonnull TriFunction<String, Object, byte[], CompletableFuture<Object>> updateAggregateFunction) {
	this.address = address;
	this.hostname = hostname;
	this.cancelFunction = cancelFunction;
	this.updateTaskExecutionStateFunction = updateTaskExecutionStateFunction;
	this.requestNextInputSplitFunction = requestNextInputSplitFunction;
	this.requestPartitionStateFunction = requestPartitionStateFunction;
	this.scheduleOrUpdateConsumersFunction = scheduleOrUpdateConsumersFunction;
	this.disconnectTaskManagerFunction = disconnectTaskManagerFunction;
	this.disconnectResourceManagerConsumer = disconnectResourceManagerConsumer;
	this.classloadingPropsSupplier = classloadingPropsSupplier;
	this.offerSlotsFunction = offerSlotsFunction;
	this.failSlotConsumer = failSlotConsumer;
	this.registerTaskManagerFunction = registerTaskManagerFunction;
	this.taskManagerHeartbeatConsumer = taskManagerHeartbeatConsumer;
	this.resourceManagerHeartbeatConsumer = resourceManagerHeartbeatConsumer;
	this.requestJobDetailsSupplier = requestJobDetailsSupplier;
	this.requestJobSupplier = requestJobSupplier;
	this.triggerSavepointFunction = triggerSavepointFunction;
	this.stopWithSavepointFunction = stopWithSavepointFunction;
	this.requestOperatorBackPressureStatsFunction = requestOperatorBackPressureStatsFunction;
	this.notifyAllocationFailureConsumer = notifyAllocationFailureConsumer;
	this.acknowledgeCheckpointConsumer = acknowledgeCheckpointConsumer;
	this.declineCheckpointConsumer = declineCheckpointConsumer;
	this.fencingTokenSupplier = fencingTokenSupplier;
	this.requestKvStateLocationFunction = requestKvStateLocationFunction;
	this.notifyKvStateRegisteredFunction = notifyKvStateRegisteredFunction;
	this.notifyKvStateUnregisteredFunction = notifyKvStateUnregisteredFunction;
	this.updateAggregateFunction = updateAggregateFunction;
}

Example #21

Source File: TestingJobMasterGatewayBuilder.java From flink with Apache License 2.0

4 votes

public TestingJobMasterGatewayBuilder setRequestKvStateLocationFunction(BiFunction<JobID, String, CompletableFuture<KvStateLocation>> requestKvStateLocationFunction) {
	this.requestKvStateLocationFunction = requestKvStateLocationFunction;
	return this;
}

Example #22

Source File: KvStateClientProxyImplTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
public CompletableFuture<KvStateLocation> requestKvStateLocation(JobID jobId, String registrationName) {
	return null;
}

Example #23

Source File: KvStateClientProxyHandler.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Lookup the {@link KvStateLocation} for the given job and queryable state name.
 *
 * <p>The job manager will be queried for the location only if forced or no
 * cached location can be found. There are no guarantees about
 *
 * @param jobId              JobID the state instance belongs to.
 * @param queryableStateName Name under which the state instance has been published.
 * @param forceUpdate        Flag to indicate whether to force a update via the lookup service.
 * @return Future holding the KvStateLocation
 */
private CompletableFuture<KvStateLocation> getKvStateLookupInfo(
		final JobID jobId,
		final String queryableStateName,
		final boolean forceUpdate) {

	final Tuple2<JobID, String> cacheKey = new Tuple2<>(jobId, queryableStateName);
	final CompletableFuture<KvStateLocation> cachedFuture = lookupCache.get(cacheKey);

	if (!forceUpdate && cachedFuture != null && !cachedFuture.isCompletedExceptionally()) {
		LOG.debug("Retrieving location for state={} of job={} from the cache.", queryableStateName, jobId);
		return cachedFuture;
	}

	final KvStateLocationOracle kvStateLocationOracle = proxy.getKvStateLocationOracle(jobId);

	if (kvStateLocationOracle != null) {
		LOG.debug("Retrieving location for state={} of job={} from the key-value state location oracle.", queryableStateName, jobId);
		final CompletableFuture<KvStateLocation> location = new CompletableFuture<>();
		lookupCache.put(cacheKey, location);

		kvStateLocationOracle
			.requestKvStateLocation(jobId, queryableStateName)
			.whenComplete(
				(KvStateLocation kvStateLocation, Throwable throwable) -> {
					if (throwable != null) {
						if (ExceptionUtils.stripCompletionException(throwable) instanceof FlinkJobNotFoundException) {
							// if the jobId was wrong, remove the entry from the cache.
							lookupCache.remove(cacheKey);
						}
						location.completeExceptionally(throwable);
					} else {
						location.complete(kvStateLocation);
					}
				});

		return location;
	} else {
		return FutureUtils.completedExceptionally(
			new UnknownLocationException(
					"Could not retrieve location of state=" + queryableStateName + " of job=" + jobId +
							". Potential reasons are: i) the state is not ready, or ii) the job does not exist."));
	}
}

Example #24

Source File: JobMasterTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testRegisterAndUnregisterKvState() throws Exception {
	final JobGraph graph = createKvJobGraph();
	final List<JobVertex> jobVertices = graph.getVerticesSortedTopologicallyFromSources();
	final JobVertex vertex1 = jobVertices.get(0);

	final JobMaster jobMaster = createJobMaster(
		configuration,
		graph,
		haServices,
		new TestingJobManagerSharedServicesBuilder().build(),
		heartbeatServices);

	CompletableFuture<Acknowledge> startFuture = jobMaster.start(jobMasterId);
	final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);

	try {
		// wait for the start to complete
		startFuture.get(testingTimeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		// register a KvState
		final String registrationName = "register-me";
		final KvStateID kvStateID = new KvStateID();
		final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
		final InetSocketAddress address = new InetSocketAddress(InetAddress.getLocalHost(), 1029);

		jobMasterGateway.notifyKvStateRegistered(
			graph.getJobID(),
			vertex1.getID(),
			keyGroupRange,
			registrationName,
			kvStateID,
			address).get();

		final KvStateLocation location = jobMasterGateway.requestKvStateLocation(graph.getJobID(), registrationName).get();

		assertEquals(graph.getJobID(), location.getJobId());
		assertEquals(vertex1.getID(), location.getJobVertexId());
		assertEquals(vertex1.getMaxParallelism(), location.getNumKeyGroups());
		assertEquals(1, location.getNumRegisteredKeyGroups());
		assertEquals(1, keyGroupRange.getNumberOfKeyGroups());
		assertEquals(kvStateID, location.getKvStateID(keyGroupRange.getStartKeyGroup()));
		assertEquals(address, location.getKvStateServerAddress(keyGroupRange.getStartKeyGroup()));

		// unregister the KvState
		jobMasterGateway.notifyKvStateUnregistered(
			graph.getJobID(),
			vertex1.getID(),
			keyGroupRange,
			registrationName).get();

		try {
			jobMasterGateway.requestKvStateLocation(graph.getJobID(), registrationName).get();
			fail("Expected to fail with an UnknownKvStateLocation.");
		} catch (Exception e) {
			assertTrue(ExceptionUtils.findThrowable(e, UnknownKvStateLocation.class).isPresent());
		}
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
	}
}

Example #25

Source File: TestingJobMasterGateway.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
public CompletableFuture<KvStateLocation> requestKvStateLocation(JobID jobId, String registrationName) {
	return requestKvStateLocationFunction.apply(jobId, registrationName);
}

Example #26

Source File: TestingJobMasterGateway.java From Flink-CEPplus with Apache License 2.0

4 votes

public TestingJobMasterGateway(
		@Nonnull String address,
		@Nonnull String hostname,
		@Nonnull Supplier<CompletableFuture<Acknowledge>> cancelFunction,
		@Nonnull Supplier<CompletableFuture<Acknowledge>> stopFunction,
		@Nonnull BiFunction<Integer, RescalingBehaviour, CompletableFuture<Acknowledge>> rescalingJobFunction,
		@Nonnull TriFunction<Collection<JobVertexID>, Integer, RescalingBehaviour, CompletableFuture<Acknowledge>> rescalingOperatorsFunction,
		@Nonnull Function<TaskExecutionState, CompletableFuture<Acknowledge>> updateTaskExecutionStateFunction,
		@Nonnull BiFunction<JobVertexID, ExecutionAttemptID, CompletableFuture<SerializedInputSplit>> requestNextInputSplitFunction,
		@Nonnull BiFunction<IntermediateDataSetID, ResultPartitionID, CompletableFuture<ExecutionState>> requestPartitionStateFunction,
		@Nonnull Function<ResultPartitionID, CompletableFuture<Acknowledge>> scheduleOrUpdateConsumersFunction,
		@Nonnull Function<ResourceID, CompletableFuture<Acknowledge>> disconnectTaskManagerFunction,
		@Nonnull Consumer<ResourceManagerId> disconnectResourceManagerConsumer,
		@Nonnull Supplier<CompletableFuture<ClassloadingProps>> classloadingPropsSupplier,
		@Nonnull BiFunction<ResourceID, Collection<SlotOffer>, CompletableFuture<Collection<SlotOffer>>> offerSlotsFunction,
		@Nonnull TriConsumer<ResourceID, AllocationID, Throwable> failSlotConsumer,
		@Nonnull BiFunction<String, TaskManagerLocation, CompletableFuture<RegistrationResponse>> registerTaskManagerFunction,
		@Nonnull BiConsumer<ResourceID, AccumulatorReport> taskManagerHeartbeatConsumer,
		@Nonnull Consumer<ResourceID> resourceManagerHeartbeatConsumer,
		@Nonnull Supplier<CompletableFuture<JobDetails>> requestJobDetailsSupplier,
		@Nonnull Supplier<CompletableFuture<ArchivedExecutionGraph>> requestJobSupplier,
		@Nonnull BiFunction<String, Boolean, CompletableFuture<String>> triggerSavepointFunction,
		@Nonnull Function<JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOperatorBackPressureStatsFunction,
		@Nonnull BiConsumer<AllocationID, Throwable> notifyAllocationFailureConsumer,
		@Nonnull Consumer<Tuple5<JobID, ExecutionAttemptID, Long, CheckpointMetrics, TaskStateSnapshot>> acknowledgeCheckpointConsumer,
		@Nonnull Consumer<DeclineCheckpoint> declineCheckpointConsumer,
		@Nonnull Supplier<JobMasterId> fencingTokenSupplier,
		@Nonnull BiFunction<JobID, String, CompletableFuture<KvStateLocation>> requestKvStateLocationFunction,
		@Nonnull Function<Tuple6<JobID, JobVertexID, KeyGroupRange, String, KvStateID, InetSocketAddress>, CompletableFuture<Acknowledge>> notifyKvStateRegisteredFunction,
		@Nonnull Function<Tuple4<JobID, JobVertexID, KeyGroupRange, String>, CompletableFuture<Acknowledge>> notifyKvStateUnregisteredFunction,
		@Nonnull TriFunction<String, Object, byte[], CompletableFuture<Object>> updateAggregateFunction) {
	this.address = address;
	this.hostname = hostname;
	this.cancelFunction = cancelFunction;
	this.stopFunction = stopFunction;
	this.rescalingJobFunction = rescalingJobFunction;
	this.rescalingOperatorsFunction = rescalingOperatorsFunction;
	this.updateTaskExecutionStateFunction = updateTaskExecutionStateFunction;
	this.requestNextInputSplitFunction = requestNextInputSplitFunction;
	this.requestPartitionStateFunction = requestPartitionStateFunction;
	this.scheduleOrUpdateConsumersFunction = scheduleOrUpdateConsumersFunction;
	this.disconnectTaskManagerFunction = disconnectTaskManagerFunction;
	this.disconnectResourceManagerConsumer = disconnectResourceManagerConsumer;
	this.classloadingPropsSupplier = classloadingPropsSupplier;
	this.offerSlotsFunction = offerSlotsFunction;
	this.failSlotConsumer = failSlotConsumer;
	this.registerTaskManagerFunction = registerTaskManagerFunction;
	this.taskManagerHeartbeatConsumer = taskManagerHeartbeatConsumer;
	this.resourceManagerHeartbeatConsumer = resourceManagerHeartbeatConsumer;
	this.requestJobDetailsSupplier = requestJobDetailsSupplier;
	this.requestJobSupplier = requestJobSupplier;
	this.triggerSavepointFunction = triggerSavepointFunction;
	this.requestOperatorBackPressureStatsFunction = requestOperatorBackPressureStatsFunction;
	this.notifyAllocationFailureConsumer = notifyAllocationFailureConsumer;
	this.acknowledgeCheckpointConsumer = acknowledgeCheckpointConsumer;
	this.declineCheckpointConsumer = declineCheckpointConsumer;
	this.fencingTokenSupplier = fencingTokenSupplier;
	this.requestKvStateLocationFunction = requestKvStateLocationFunction;
	this.notifyKvStateRegisteredFunction = notifyKvStateRegisteredFunction;
	this.notifyKvStateUnregisteredFunction = notifyKvStateUnregisteredFunction;
	this.updateAggregateFunction = updateAggregateFunction;
}

Example #27

Source File: TestingJobMasterGatewayBuilder.java From Flink-CEPplus with Apache License 2.0

4 votes

public TestingJobMasterGatewayBuilder setRequestKvStateLocationFunction(BiFunction<JobID, String, CompletableFuture<KvStateLocation>> requestKvStateLocationFunction) {
	this.requestKvStateLocationFunction = requestKvStateLocationFunction;
	return this;
}

Example #28

Source File: KvStateLocationOracle.java From flink with Apache License 2.0

2 votes

/**
 * Requests a {@link KvStateLocation} for the specified {@link InternalKvState} registration name.
 *
 * @param jobId identifying the job for which to request the {@link KvStateLocation}
 * @param registrationName Name under which the KvState has been registered.
 * @return Future of the requested {@link InternalKvState} location
 */
CompletableFuture<KvStateLocation> requestKvStateLocation(
	final JobID jobId,
	final String registrationName);

Example #29

Source File: KvStateLocationOracle.java From flink with Apache License 2.0

2 votes

/**
 * Requests a {@link KvStateLocation} for the specified {@link InternalKvState} registration name.
 *
 * @param jobId identifying the job for which to request the {@link KvStateLocation}
 * @param registrationName Name under which the KvState has been registered.
 * @return Future of the requested {@link InternalKvState} location
 */
CompletableFuture<KvStateLocation> requestKvStateLocation(
	final JobID jobId,
	final String registrationName);

Example #30

Source File: KvStateLocationOracle.java From Flink-CEPplus with Apache License 2.0

2 votes

/**
 * Requests a {@link KvStateLocation} for the specified {@link InternalKvState} registration name.
 *
 * @param jobId identifying the job for which to request the {@link KvStateLocation}
 * @param registrationName Name under which the KvState has been registered.
 * @return Future of the requested {@link InternalKvState} location
 */
CompletableFuture<KvStateLocation> requestKvStateLocation(
	final JobID jobId,
	final String registrationName);