Java Code Examples for org.apache.flink.runtime.clusterframework.types.ResourceProfile#ZERO

private TaskExecutorToResourceManagerConnection createTaskExecutorToResourceManagerConnection() {
	final TaskExecutorRegistration taskExecutorRegistration = new TaskExecutorRegistration(
		TASK_MANAGER_ADDRESS,
		TASK_MANAGER_RESOURCE_ID,
		TASK_MANAGER_DATA_PORT,
		TASK_MANAGER_HARDWARE_DESCRIPTION,
		ResourceProfile.ZERO,
		ResourceProfile.ZERO
	);
	return new TaskExecutorToResourceManagerConnection(
		LOGGER,
		rpcService,
		RetryingRegistrationConfiguration.defaultConfiguration(),
		RESOURCE_MANAGER_ADDRESS,
		RESOURCE_MANAGER_ID,
		Executors.directExecutor(),
		new TestRegistrationConnectionListener<>(),
		taskExecutorRegistration);
}
 

static TaskManagerInfo createRandomTaskManagerInfo() {
	return new TaskManagerInfo(
		ResourceID.generate(),
		UUID.randomUUID().toString(),
		random.nextInt(),
		random.nextLong(),
		random.nextInt(),
		random.nextInt(),
		ResourceProfile.ZERO,
		ResourceProfile.ZERO,
		new HardwareDescription(
			random.nextInt(),
			random.nextLong(),
			random.nextLong(),
			random.nextLong()));
}
 

private MultiTaskSlot(
		SlotRequestId slotRequestId,
		@Nullable AbstractID groupId,
		@Nullable MultiTaskSlot parent,
		CompletableFuture<? extends SlotContext> slotContextFuture,
		@Nullable SlotRequestId allocatedSlotRequestId) {
	super(slotRequestId, groupId);
	Preconditions.checkNotNull(slotContextFuture);

	this.parent = parent;
	this.allocatedSlotRequestId = allocatedSlotRequestId;

	this.children = new HashMap<>(16);
	this.releasingChildren = false;

	this.reservedResources = ResourceProfile.ZERO;

	this.slotContextFuture = slotContextFuture.handle((SlotContext slotContext, Throwable throwable) -> {
		if (throwable != null) {
			// If the underlying resource request failed, we currently fail all the requests
			release(throwable);
			throw new CompletionException(throwable);
		}

		if (parent == null) {
			checkOversubscriptionAndReleaseChildren(slotContext);
		}

		return slotContext;
	});
}
 

/**
 * Test worker registration after launch.
 */
@Test
public void testWorkerStarted() throws Exception {
	new Context() {{
		// set the initial state with a (recovered) launched worker
		MesosWorkerStore.Worker worker1launched = MesosWorkerStore.Worker.newWorker(task1, workerResourceSpec).launchWorker(slave1, slave1host);
		when(rmServices.workerStore.getFrameworkID()).thenReturn(Option.apply(framework1));
		when(rmServices.workerStore.recoverWorkers()).thenReturn(singletonList(worker1launched));
		startResourceManager();
		assertThat(resourceManager.workersInLaunch, hasEntry(extractResourceID(task1), worker1launched));

		final int dataPort = 1234;
		final HardwareDescription hardwareDescription = new HardwareDescription(1, 2L, 3L, 4L);
		// send registration message
		TaskExecutorRegistration taskExecutorRegistration = new TaskExecutorRegistration(
			task1Executor.address,
			task1Executor.resourceID,
			dataPort,
			hardwareDescription,
			ResourceProfile.ZERO,
			ResourceProfile.ZERO);
		CompletableFuture<RegistrationResponse> successfulFuture =
			resourceManager.registerTaskExecutor(taskExecutorRegistration, timeout);
		RegistrationResponse response = successfulFuture.get(timeout.toMilliseconds(), TimeUnit.MILLISECONDS);
		assertTrue(response instanceof TaskExecutorRegistrationSuccess);
		final TaskExecutorRegistrationSuccess registrationResponse = (TaskExecutorRegistrationSuccess) response;

		final CompletableFuture<Acknowledge> initialSlotReportFuture = resourceManager.sendSlotReport(task1Executor.resourceID, registrationResponse.getRegistrationId(), slotReport, timeout);

		// check for errors
		initialSlotReportFuture.get();

		// verify the internal state
		assertThat(resourceManager.workersInLaunch, hasEntry(extractResourceID(task1), worker1launched));
	}};
}
 

private MultiTaskSlot(
		SlotRequestId slotRequestId,
		@Nullable AbstractID groupId,
		@Nullable MultiTaskSlot parent,
		CompletableFuture<? extends SlotContext> slotContextFuture,
		@Nullable SlotRequestId allocatedSlotRequestId) {
	super(slotRequestId, groupId);
	Preconditions.checkNotNull(slotContextFuture);

	this.parent = parent;
	this.allocatedSlotRequestId = allocatedSlotRequestId;

	this.children = new HashMap<>(16);
	this.releasingChildren = false;

	this.reservedResources = ResourceProfile.ZERO;

	this.slotContextFuture = slotContextFuture.handle((SlotContext slotContext, Throwable throwable) -> {
		if (throwable != null) {
			// If the underlying resource request failed, we currently fail all the requests
			release(throwable);
			throw new CompletionException(throwable);
		}

		if (parent == null) {
			// sanity check
			releaseSlotIfOversubscribing(slotContext);
		}

		return slotContext;
	});
}
 

private void registerTaskExecutor(ResourceManagerGateway resourceManagerGateway, ResourceID taskExecutorId, String taskExecutorAddress) throws Exception {
	TaskExecutorRegistration taskExecutorRegistration = new TaskExecutorRegistration(
		taskExecutorAddress,
		taskExecutorId,
		dataPort,
		hardwareDescription,
		ResourceProfile.ZERO,
		ResourceProfile.ZERO);
	final CompletableFuture<RegistrationResponse> registrationFuture = resourceManagerGateway.registerTaskExecutor(
		taskExecutorRegistration,
		TestingUtils.TIMEOUT());

	assertThat(registrationFuture.get(), instanceOf(RegistrationResponse.Success.class));
}
 

private void registerTaskExecutor(ResourceManagerGateway resourceManagerGateway, ResourceID taskExecutorId, String taskExecutorAddress) throws Exception {
	final TaskExecutorRegistration taskExecutorRegistration = new TaskExecutorRegistration(
		taskExecutorAddress,
		taskExecutorId,
		1234,
		new HardwareDescription(42, 1337L, 1337L, 0L),
		ResourceProfile.ZERO,
		ResourceProfile.ZERO);
	final CompletableFuture<RegistrationResponse> registrationFuture = resourceManagerGateway.registerTaskExecutor(
		taskExecutorRegistration,
		TestingUtils.TIMEOUT());

	assertThat(registrationFuture.get(), instanceOf(RegistrationResponse.Success.class));
}
 

private void checkOversubscriptionAndReleaseChildren(SlotContext slotContext) {
	final ResourceProfile slotResources = slotContext.getResourceProfile();
	final ArrayList<TaskSlot> childrenToEvict = new ArrayList<>();
	ResourceProfile requiredResources = ResourceProfile.ZERO;

	for (TaskSlot slot : children.values()) {
		final ResourceProfile resourcesWithChild = requiredResources.merge(slot.getReservedResources());

		if (slotResources.isMatching(resourcesWithChild)) {
			requiredResources = resourcesWithChild;
		} else {
			childrenToEvict.add(slot);
		}
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("Not all requests are fulfilled due to over-allocated, number of requests is {}, " +
				"number of evicted requests is {}, underlying allocated is {}, fulfilled is {}, " +
				"evicted requests is {},",
			children.size(),
			childrenToEvict.size(),
			slotContext.getResourceProfile(),
			requiredResources,
			childrenToEvict);
	}

	if (childrenToEvict.size() == children.size()) {
		// Since RM always return a slot whose resource is larger than the requested one,
		// The current situation only happens when we request to RM using the resource
		// profile of a task who is belonging to a CoLocationGroup. Similar to dealing
		// with the failure of the underlying request, currently we fail all the requests
		// directly.
		release(new SharedSlotOversubscribedException(
			"The allocated slot does not have enough resource for any task.", false));
	} else {
		for (TaskSlot taskSlot : childrenToEvict) {
			taskSlot.release(new SharedSlotOversubscribedException(
				"The allocated slot does not have enough resource for all the tasks.", true));
		}
	}
}
 

private static <T extends TaskSlotPayload> TaskSlot<T> createTaskSlot() {
	return new TaskSlot<>(0, ResourceProfile.ZERO, MemoryManager.MIN_PAGE_SIZE, JOB_ID, ALLOCATION_ID);
}
 

/**
 * Test delayed registration of task executor where the delay is introduced during connection from resource manager
 * to the registering task executor.
 */
@Test
public void testDelayedRegisterTaskExecutor() throws Exception {
	final Time fastTimeout = Time.milliseconds(1L);
	try {
		final OneShotLatch startConnection = new OneShotLatch();
		final OneShotLatch finishConnection = new OneShotLatch();

		// first registration is with blocking connection
		rpcService.setRpcGatewayFutureFunction(rpcGateway ->
			CompletableFuture.supplyAsync(
				() -> {
					startConnection.trigger();
					try {
						finishConnection.await();
					} catch (InterruptedException ignored) {}
					return rpcGateway;
				},
				TestingUtils.defaultExecutor()));

		TaskExecutorRegistration taskExecutorRegistration = new TaskExecutorRegistration(
			taskExecutorGateway.getAddress(),
			taskExecutorResourceID,
			dataPort,
			hardwareDescription,
			ResourceProfile.ZERO,
			ResourceProfile.ZERO);

		CompletableFuture<RegistrationResponse> firstFuture =
			rmGateway.registerTaskExecutor(taskExecutorRegistration, fastTimeout);
		try {
			firstFuture.get();
			fail("Should have failed because connection to taskmanager is delayed beyond timeout");
		} catch (Exception e) {
			final Throwable cause = ExceptionUtils.stripExecutionException(e);
			assertThat(cause, instanceOf(TimeoutException.class));
			assertThat(cause.getMessage(), containsString("ResourceManagerGateway.registerTaskExecutor"));
		}

		startConnection.await();

		// second registration after timeout is with no delay, expecting it to be succeeded
		rpcService.resetRpcGatewayFutureFunction();
		CompletableFuture<RegistrationResponse> secondFuture =
			rmGateway.registerTaskExecutor(taskExecutorRegistration, TIMEOUT);
		RegistrationResponse response = secondFuture.get();
		assertTrue(response instanceof TaskExecutorRegistrationSuccess);

		// on success, send slot report for taskmanager registration
		final SlotReport slotReport = new SlotReport(new SlotStatus(new SlotID(taskExecutorResourceID, 0), ResourceProfile.ANY));
		rmGateway.sendSlotReport(taskExecutorResourceID,
			((TaskExecutorRegistrationSuccess) response).getRegistrationId(), slotReport, TIMEOUT).get();

		// let the remaining part of the first registration proceed
		finishConnection.trigger();
		Thread.sleep(1L);

		// verify that the latest registration is valid not being unregistered by the delayed one
		final TaskManagerInfo taskManagerInfo = rmGateway.requestTaskManagerInfo(
			taskExecutorResourceID,
			TIMEOUT).get();
		assertThat(taskManagerInfo.getResourceId(), equalTo(taskExecutorResourceID));
		assertThat(taskManagerInfo.getNumberSlots(), equalTo(1));
	} finally {
		rpcService.resetRpcGatewayFutureFunction();
	}
}
 

@Override
public ResourceProfile getRegisteredResource() {
	return ResourceProfile.ZERO;
}
 

@Override
public ResourceProfile getRegisteredResourceOf(InstanceID instanceID) {
	return ResourceProfile.ZERO;
}
 

@Override
public ResourceProfile getFreeResource() {
	return ResourceProfile.ZERO;
}
 

@Override
public ResourceProfile getFreeResourceOf(InstanceID instanceID) {
	return ResourceProfile.ZERO;
}