Java Code Examples for org.apache.flink.runtime.taskmanager.TaskManagerLocation

The following examples show how to use org.apache.flink.runtime.taskmanager.TaskManagerLocation. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: Flink-CEPplus   Author: ljygz   File: ExecutionGraphSchedulingTest.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
static SingleLogicalSlot createSingleLogicalSlot(SlotOwner slotOwner, TaskManagerGateway taskManagerGateway, SlotRequestId slotRequestId) {
	TaskManagerLocation location = new TaskManagerLocation(
		ResourceID.generate(), InetAddress.getLoopbackAddress(), 12345);

	SimpleSlotContext slotContext = new SimpleSlotContext(
		new AllocationID(),
		location,
		0,
		taskManagerGateway);

	return new SingleLogicalSlot(
		slotRequestId,
		slotContext,
		null,
		Locality.LOCAL,
		slotOwner);
}
 
Example #2
Source Project: Flink-CEPplus   Author: ljygz   File: PartialInputChannelDeploymentDescriptor.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a channel deployment descriptor by completing the partition location.
 *
 * @see InputChannelDeploymentDescriptor
 */
public InputChannelDeploymentDescriptor createInputChannelDeploymentDescriptor(Execution consumerExecution) {
	checkNotNull(consumerExecution, "consumerExecution");

	TaskManagerLocation consumerLocation = consumerExecution.getAssignedResourceLocation();
	checkNotNull(consumerLocation, "Consumer connection info null");

	final ResultPartitionLocation partitionLocation;

	if (consumerLocation.equals(partitionTaskManagerLocation)) {
		partitionLocation = ResultPartitionLocation.createLocal();
	}
	else {
		partitionLocation = ResultPartitionLocation.createRemote(
				new ConnectionID(partitionTaskManagerLocation, partitionConnectionIndex));
	}

	return new InputChannelDeploymentDescriptor(partitionID, partitionLocation);
}
 
Example #3
Source Project: flink   Author: apache   File: DefaultPreferredLocationsRetriever.java    License: Apache License 2.0 6 votes vote down vote up
private CompletableFuture<Collection<TaskManagerLocation>> combineLocations(
		final CompletableFuture<Collection<TaskManagerLocation>> locationsCombinedAlready,
		final Collection<CompletableFuture<TaskManagerLocation>> locationsToCombine) {

	final CompletableFuture<Set<TaskManagerLocation>> uniqueLocationsFuture =
		FutureUtils.combineAll(locationsToCombine).thenApply(HashSet::new);

	return locationsCombinedAlready.thenCombine(
		uniqueLocationsFuture,
		(locationsOnOneEdge, locationsOnAnotherEdge) -> {
			if ((!locationsOnOneEdge.isEmpty() && locationsOnAnotherEdge.size() > locationsOnOneEdge.size())
				|| locationsOnAnotherEdge.isEmpty()) {
				return locationsOnOneEdge;
			} else {
				return locationsOnAnotherEdge;
			}
		});
}
 
Example #4
Source Project: flink   Author: apache   File: SimpleSlotProvider.java    License: Apache License 2.0 6 votes vote down vote up
public SimpleSlotProvider(int numSlots, TaskManagerGateway taskManagerGateway) {
	checkArgument(numSlots >= 0, "numSlots must be >= 0");

	this.slots = new ArrayDeque<>(numSlots);

	for (int i = 0; i < numSlots; i++) {
		SimpleSlotContext as = new SimpleSlotContext(
			new AllocationID(),
			new TaskManagerLocation(ResourceID.generate(), InetAddress.getLoopbackAddress(), 10000 + i),
			0,
			taskManagerGateway,
			ResourceProfile.ANY);
		slots.add(as);
	}

	allocatedSlots = new HashMap<>(slots.size());
}
 
Example #5
Source Project: flink   Author: flink-tpc-ds   File: ExecutionGraphToInputsLocationsRetrieverAdapterTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that it can get the task manager location in an Execution.
 */
@Test
public void testGetTaskManagerLocationWhenScheduled() throws Exception {
	final JobVertex jobVertex = ExecutionGraphTestUtils.createNoOpVertex(1);

	final TestingLogicalSlot testingLogicalSlot = new TestingLogicalSlotBuilder().createTestingLogicalSlot();
	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(new JobID(), jobVertex);
	final ExecutionGraphToInputsLocationsRetrieverAdapter inputsLocationsRetriever =
			new ExecutionGraphToInputsLocationsRetrieverAdapter(eg);

	final ExecutionVertex onlyExecutionVertex = eg.getAllExecutionVertices().iterator().next();
	onlyExecutionVertex.deployToSlot(testingLogicalSlot);

	ExecutionVertexID executionVertexId = new ExecutionVertexID(jobVertex.getID(), 0);
	Optional<CompletableFuture<TaskManagerLocation>> taskManagerLocationOptional =
			inputsLocationsRetriever.getTaskManagerLocation(executionVertexId);

	assertTrue(taskManagerLocationOptional.isPresent());

	final CompletableFuture<TaskManagerLocation> taskManagerLocationFuture = taskManagerLocationOptional.get();
	assertThat(taskManagerLocationFuture.get(), is(testingLogicalSlot.getTaskManagerLocation()));
}
 
Example #6
Source Project: flink   Author: apache   File: DefaultPreferredLocationsRetriever.java    License: Apache License 2.0 6 votes vote down vote up
private CompletableFuture<Collection<TaskManagerLocation>> getPreferredLocationsBasedOnInputs(
		final ExecutionVertexID executionVertexId,
		final Set<ExecutionVertexID> producersToIgnore) {

	CompletableFuture<Collection<TaskManagerLocation>> preferredLocations =
		CompletableFuture.completedFuture(Collections.emptyList());

	final Collection<Collection<ExecutionVertexID>> allProducers =
		inputsLocationsRetriever.getConsumedResultPartitionsProducers(executionVertexId);
	for (Collection<ExecutionVertexID> producers : allProducers) {
		final Collection<CompletableFuture<TaskManagerLocation>> locationsFutures =
			getInputLocationFutures(producersToIgnore, producers);

		preferredLocations = combineLocations(preferredLocations, locationsFutures);
	}
	return preferredLocations;
}
 
Example #7
Source Project: Flink-CEPplus   Author: ljygz   File: ExecutionVertexLocalityTest.java    License: Apache License 2.0 6 votes vote down vote up
private void initializeLocation(ExecutionVertex vertex, TaskManagerLocation location) throws Exception {
	// we need a bit of reflection magic to initialize the location without going through
	// scheduling paths. we choose to do that, rather than the alternatives:
	//  - mocking the scheduler created fragile tests that break whenever the scheduler is adjusted
	//  - exposing test methods in the ExecutionVertex leads to undesirable setters 

	SlotContext slot = new SimpleSlotContext(
		new AllocationID(),
		location,
		0,
		mock(TaskManagerGateway.class));

	SimpleSlot simpleSlot = new SimpleSlot(slot, mock(SlotOwner.class), 0);

	if (!vertex.getCurrentExecutionAttempt().tryAssignResource(simpleSlot)) {
		throw new FlinkException("Could not assign resource.");
	}
}
 
Example #8
Source Project: flink   Author: apache   File: LocationPreferenceSlotSelectionStrategy.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Optional<SlotInfoAndLocality> selectBestSlotForProfile(
	@Nonnull Collection<SlotInfoAndResources> availableSlots,
	@Nonnull SlotProfile slotProfile) {

	Collection<TaskManagerLocation> locationPreferences = slotProfile.getPreferredLocations();

	if (availableSlots.isEmpty()) {
		return Optional.empty();
	}

	final ResourceProfile resourceProfile = slotProfile.getPhysicalSlotResourceProfile();

	// if we have no location preferences, we can only filter by the additional requirements.
	return locationPreferences.isEmpty() ?
		selectWithoutLocationPreference(availableSlots, resourceProfile) :
		selectWitLocationPreference(availableSlots, locationPreferences, resourceProfile);
}
 
Example #9
Source Project: Flink-CEPplus   Author: ljygz   File: JobMaster.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Collection<SlotOffer>> offerSlots(
		final ResourceID taskManagerId,
		final Collection<SlotOffer> slots,
		final Time timeout) {

	Tuple2<TaskManagerLocation, TaskExecutorGateway> taskManager = registeredTaskManagers.get(taskManagerId);

	if (taskManager == null) {
		return FutureUtils.completedExceptionally(new Exception("Unknown TaskManager " + taskManagerId));
	}

	final TaskManagerLocation taskManagerLocation = taskManager.f0;
	final TaskExecutorGateway taskExecutorGateway = taskManager.f1;

	final RpcTaskManagerGateway rpcTaskManagerGateway = new RpcTaskManagerGateway(taskExecutorGateway, getFencingToken());

	return CompletableFuture.completedFuture(
		slotPool.offerSlots(
			taskManagerLocation,
			rpcTaskManagerGateway,
			slots));
}
 
Example #10
Source Project: Flink-CEPplus   Author: ljygz   File: SimpleSlot.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new simple slot that belongs to the given shared slot and
 * is identified by the given ID.
 *
 * @param owner The component from which this slot is allocated.
 * @param location The location info of the TaskManager where the slot was allocated from
 * @param slotNumber The number of the simple slot in its parent shared slot.
 * @param taskManagerGateway to communicate with the associated task manager.
 * @param parent The parent shared slot.
 * @param groupID The ID that identifies the group that the slot belongs to.
 */
public SimpleSlot(
		SlotOwner owner,
		TaskManagerLocation location,
		int slotNumber,
		TaskManagerGateway taskManagerGateway,
		@Nullable SharedSlot parent,
		@Nullable AbstractID groupID) {

	super(
		parent != null ?
			parent.getSlotContext() :
			new SimpleSlotContext(
				NO_ALLOCATION_ID,
				location,
				slotNumber,
				taskManagerGateway),
		owner,
		slotNumber,
		parent,
		groupID);
}
 
Example #11
Source Project: flink   Author: apache   File: ExecutionTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that any preferred locations are calculated.
 */
@Test
public void testAnyPreferredLocationCalculation() throws Exception {
	final TaskManagerLocation taskManagerLocation1 = new LocalTaskManagerLocation();
	final TaskManagerLocation taskManagerLocation3 = new LocalTaskManagerLocation();

	final CompletableFuture<TaskManagerLocation> locationFuture1 = CompletableFuture.completedFuture(taskManagerLocation1);
	final CompletableFuture<TaskManagerLocation> locationFuture2 = new CompletableFuture<>();
	final CompletableFuture<TaskManagerLocation> locationFuture3 = CompletableFuture.completedFuture(taskManagerLocation3);

	final Execution execution = getExecution(Arrays.asList(locationFuture1, locationFuture2, locationFuture3));

	CompletableFuture<Collection<TaskManagerLocation>> preferredLocationsFuture = execution.calculatePreferredLocations(LocationPreferenceConstraint.ANY);

	assertTrue(preferredLocationsFuture.isDone());

	final Collection<TaskManagerLocation> preferredLocations = preferredLocationsFuture.get();

	assertThat(preferredLocations, containsInAnyOrder(taskManagerLocation1, taskManagerLocation3));
}
 
Example #12
Source Project: flink   Author: apache   File: StreamNetworkBenchmarkEnvironment.java    License: Apache License 2.0 6 votes vote down vote up
private InputGateDeploymentDescriptor createInputGateDeploymentDescriptor(
		TaskManagerLocation senderLocation,
		int gateIndex,
		ResourceID localLocation) {

	final ShuffleDescriptor[] channelDescriptors = new ShuffleDescriptor[channels];
	for (int channelIndex = 0; channelIndex < channels; ++channelIndex) {
		channelDescriptors[channelIndex] = createShuffleDescriptor(
			localMode, partitionIds[gateIndex], localLocation, senderLocation, channelIndex);
	}

	return new InputGateDeploymentDescriptor(
		dataSetID,
		ResultPartitionType.PIPELINED_BOUNDED,
		// 0 is used because TestRemoteInputChannel and TestLocalInputChannel will
		// ignore this and use channelIndex instead when requesting a subpartition
		0,
		channelDescriptors);
}
 
Example #13
Source Project: flink   Author: flink-tpc-ds   File: ExecutionVertexLocalityTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This test validates that vertices with too many input streams do not have a location
 * preference any more.
 */
@Test
public void testNoLocalityInputLargeAllToAll() throws Exception {
	final int parallelism = 100;

	final ExecutionGraph graph = createTestGraph(parallelism, true);

	// set the location for all sources to a distinct location
	for (int i = 0; i < parallelism; i++) {
		ExecutionVertex source = graph.getAllVertices().get(sourceVertexId).getTaskVertices()[i];
		TaskManagerLocation location = new TaskManagerLocation(
				ResourceID.generate(), InetAddress.getLoopbackAddress(), 10000 + i);
		initializeLocation(source, location);
	}

	// validate that the target vertices have no location preference
	for (int i = 0; i < parallelism; i++) {
		ExecutionVertex target = graph.getAllVertices().get(targetVertexId).getTaskVertices()[i];

		Iterator<CompletableFuture<TaskManagerLocation>> preference = target.getPreferredLocations().iterator();
		assertFalse(preference.hasNext());
	}
}
 
Example #14
Source Project: flink   Author: apache   File: ArchivedExecution.java    License: Apache License 2.0 6 votes vote down vote up
public ArchivedExecution(
		StringifiedAccumulatorResult[] userAccumulators, IOMetrics ioMetrics,
		ExecutionAttemptID attemptId, int attemptNumber, ExecutionState state, String failureCause,
		TaskManagerLocation assignedResourceLocation, AllocationID assignedAllocationID,  int parallelSubtaskIndex,
		long[] stateTimestamps) {
	this.userAccumulators = userAccumulators;
	this.ioMetrics = ioMetrics;
	this.failureCause = failureCause;
	this.assignedResourceLocation = assignedResourceLocation;
	this.attemptNumber = attemptNumber;
	this.attemptId = attemptId;
	this.state = state;
	this.stateTimestamps = stateTimestamps;
	this.parallelSubtaskIndex = parallelSubtaskIndex;
	this.assignedAllocationID = assignedAllocationID;
}
 
Example #15
Source Project: Flink-CEPplus   Author: ljygz   File: SimpleSlotProvider.java    License: Apache License 2.0 6 votes vote down vote up
public SimpleSlotProvider(JobID jobId, int numSlots, TaskManagerGateway taskManagerGateway) {
	checkNotNull(jobId, "jobId");
	checkArgument(numSlots >= 0, "numSlots must be >= 0");

	this.slots = new ArrayDeque<>(numSlots);

	for (int i = 0; i < numSlots; i++) {
		SimpleSlotContext as = new SimpleSlotContext(
			new AllocationID(),
			new TaskManagerLocation(ResourceID.generate(), InetAddress.getLoopbackAddress(), 10000 + i),
			0,
			taskManagerGateway);
		slots.add(as);
	}

	allocatedSlots = new HashMap<>(slots.size());
}
 
Example #16
Source Project: flink   Author: apache   File: TestingLogicalSlot.java    License: Apache License 2.0 6 votes vote down vote up
TestingLogicalSlot(
		TaskManagerLocation taskManagerLocation,
		TaskManagerGateway taskManagerGateway,
		int slotNumber,
		AllocationID allocationId,
		SlotRequestId slotRequestId,
		@Nullable SlotSharingGroupId slotSharingGroupId,
		boolean automaticallyCompleteReleaseFuture,
		SlotOwner slotOwner) {

	this.taskManagerLocation = Preconditions.checkNotNull(taskManagerLocation);
	this.taskManagerGateway = Preconditions.checkNotNull(taskManagerGateway);
	this.payloadReference = new AtomicReference<>();
	this.slotNumber = slotNumber;
	this.allocationId = Preconditions.checkNotNull(allocationId);
	this.slotRequestId = Preconditions.checkNotNull(slotRequestId);
	this.slotSharingGroupId = slotSharingGroupId;
	this.releaseFuture = new CompletableFuture<>();
	this.automaticallyCompleteReleaseFuture = automaticallyCompleteReleaseFuture;
	this.slotOwner = Preconditions.checkNotNull(slotOwner);
}
 
Example #17
Source Project: flink   Author: apache   File: ExecutionVertexLocalityTest.java    License: Apache License 2.0 6 votes vote down vote up
private void initializeLocation(ExecutionVertex vertex, TaskManagerLocation location) throws Exception {
	// we need a bit of reflection magic to initialize the location without going through
	// scheduling paths. we choose to do that, rather than the alternatives:
	//  - mocking the scheduler created fragile tests that break whenever the scheduler is adjusted
	//  - exposing test methods in the ExecutionVertex leads to undesirable setters 

	SlotContext slotContext = new SimpleSlotContext(
		new AllocationID(),
		location,
		0,
		mock(TaskManagerGateway.class));



	LogicalSlot slot = new SingleLogicalSlot(
		new SlotRequestId(),
		slotContext,
		null,
		Locality.LOCAL,
		mock(SlotOwner.class));

	if (!vertex.getCurrentExecutionAttempt().tryAssignResource(slot)) {
		throw new FlinkException("Could not assign resource.");
	}
}
 
Example #18
Source Project: flink   Author: flink-tpc-ds   File: JobLeaderService.java    License: Apache License 2.0 6 votes vote down vote up
JobManagerRetryingRegistration(
		Logger log,
		RpcService rpcService,
		String targetName,
		Class<JobMasterGateway> targetType,
		String targetAddress,
		JobMasterId jobMasterId,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		String taskManagerRpcAddress,
		TaskManagerLocation taskManagerLocation) {
	super(
		log,
		rpcService,
		targetName,
		targetType,
		targetAddress,
		jobMasterId,
		retryingRegistrationConfiguration);

	this.taskManagerRpcAddress = taskManagerRpcAddress;
	this.taskManagerLocation = Preconditions.checkNotNull(taskManagerLocation);
}
 
Example #19
Source Project: flink   Author: apache   File: SlotProfile.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns a slot profile for the given resource profile and the preferred locations.
 *
 * @param resourceProfile specifying the slot requirements
 * @param preferredLocations specifying the preferred locations
 * @return Slot profile with the given resource profile and preferred locations
 */
@VisibleForTesting
public static SlotProfile preferredLocality(
		final ResourceProfile resourceProfile,
		final Collection<TaskManagerLocation> preferredLocations) {

	return priorAllocation(
		resourceProfile,
		resourceProfile,
		preferredLocations,
		Collections.emptyList(),
		Collections.emptySet());
}
 
Example #20
Source Project: flink   Author: flink-tpc-ds   File: ProducerDescriptor.java    License: Apache License 2.0 5 votes vote down vote up
public static ProducerDescriptor create(TaskManagerLocation producerLocation, ExecutionAttemptID attemptId) {
	return new ProducerDescriptor(
		producerLocation.getResourceID(),
		attemptId,
		producerLocation.address(),
		producerLocation.dataPort());
}
 
Example #21
Source Project: flink   Author: flink-tpc-ds   File: SubtasksTimesHandler.java    License: Apache License 2.0 5 votes vote down vote up
private static SubtasksTimesInfo createSubtaskTimesInfo(AccessExecutionJobVertex jobVertex) {
	final String id = jobVertex.getJobVertexId().toString();
	final String name = jobVertex.getName();
	final long now = System.currentTimeMillis();
	final List<SubtasksTimesInfo.SubtaskTimeInfo> subtasks = new ArrayList<>();

	int num = 0;
	for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) {

		long[] timestamps = vertex.getCurrentExecutionAttempt().getStateTimestamps();
		ExecutionState status = vertex.getExecutionState();

		long scheduledTime = timestamps[ExecutionState.SCHEDULED.ordinal()];

		long start = scheduledTime > 0 ? scheduledTime : -1;
		long end = status.isTerminal() ? timestamps[status.ordinal()] : now;
		long duration = start >= 0 ? end - start : -1L;

		TaskManagerLocation location = vertex.getCurrentAssignedResourceLocation();
		String locationString = location == null ? "(unassigned)" : location.getHostname();

		Map<ExecutionState, Long> timestampMap = new HashMap<>(ExecutionState.values().length);
		for (ExecutionState state : ExecutionState.values()) {
			timestampMap.put(state, timestamps[state.ordinal()]);
		}

		subtasks.add(new SubtasksTimesInfo.SubtaskTimeInfo(
			num++,
			locationString,
			duration,
			timestampMap));
	}
	return new SubtasksTimesInfo(id, name, now, subtasks);
}
 
Example #22
Source Project: flink   Author: apache   File: SlotPoolUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static ResourceID offerSlots(
		SlotPoolImpl slotPool,
		ComponentMainThreadExecutor mainThreadExecutor,
		List<ResourceProfile> resourceProfiles,
		TaskManagerGateway taskManagerGateway) {
	final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
	CompletableFuture.runAsync(
		() -> {
			slotPool.registerTaskManager(taskManagerLocation.getResourceID());

			final Collection<SlotOffer> slotOffers = IntStream
				.range(0, resourceProfiles.size())
				.mapToObj(i -> new SlotOffer(new AllocationID(), i, resourceProfiles.get(i)))
				.collect(Collectors.toList());

			final Collection<SlotOffer> acceptedOffers = slotPool.offerSlots(
				taskManagerLocation,
				taskManagerGateway,
				slotOffers);

			assertThat(acceptedOffers, is(slotOffers));
		},
		mainThreadExecutor
	).join();

	return taskManagerLocation.getResourceID();
}
 
Example #23
Source Project: flink   Author: apache   File: ExecutionVertexLocalityTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This test validates that vertices that have only one input stream try to
 * co-locate their tasks with the producer.
 */
@Test
public void testLocalityInputBasedForward() throws Exception {
	final int parallelism = 10;
	final TaskManagerLocation[] locations = new TaskManagerLocation[parallelism];

	final ExecutionGraph graph = createTestGraph(parallelism, false);

	// set the location for all sources to a distinct location
	for (int i = 0; i < parallelism; i++) {
		ExecutionVertex source = graph.getAllVertices().get(sourceVertexId).getTaskVertices()[i];

		TaskManagerLocation location = new TaskManagerLocation(
				ResourceID.generate(), InetAddress.getLoopbackAddress(), 10000 + i);

		locations[i] = location;
		initializeLocation(source, location);
	}

	// validate that the target vertices have no location preference
	for (int i = 0; i < parallelism; i++) {
		ExecutionVertex target = graph.getAllVertices().get(targetVertexId).getTaskVertices()[i];
		Iterator<CompletableFuture<TaskManagerLocation>> preference = target.getPreferredLocations().iterator();

		assertTrue(preference.hasNext());
		assertEquals(locations[i], preference.next().get());
		assertFalse(preference.hasNext());
	}
}
 
Example #24
Source Project: flink   Author: flink-tpc-ds   File: ScheduleWithCoLocationHintTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void nonColocationFollowsCoLocation() throws Exception {
	JobVertexID jid1 = new JobVertexID();
	JobVertexID jid2 = new JobVertexID();

	TaskManagerLocation loc1 = testingSlotProvider.addTaskManager(1);
	TaskManagerLocation loc2 = testingSlotProvider.addTaskManager(1);

	assertEquals(2, testingSlotProvider.getNumberOfAvailableSlots());

	SlotSharingGroup sharingGroup = new SlotSharingGroup();

	CoLocationGroup ccg = new CoLocationGroup();
	CoLocationConstraint cc1 = new CoLocationConstraint(ccg);
	CoLocationConstraint cc2 = new CoLocationConstraint(ccg);

	LogicalSlot s1 = testingSlotProvider.allocateSlot(
			new ScheduledUnit(getTestVertexWithLocation(jid1, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId(), cc1), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get();
	LogicalSlot s2 = testingSlotProvider.allocateSlot(
			new ScheduledUnit(getTestVertexWithLocation(jid1, 1, 2, sharingGroup, loc2), sharingGroup.getSlotSharingGroupId(), cc2), false, slotProfileForLocation(loc2), TestingUtils.infiniteTime()).get();

	LogicalSlot s3 = testingSlotProvider.allocateSlot(
			new ScheduledUnit(getTestVertexWithLocation(jid2, 0, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get();
	LogicalSlot s4 = testingSlotProvider.allocateSlot(
			new ScheduledUnit(getTestVertexWithLocation(jid2, 1, 2, sharingGroup, loc1), sharingGroup.getSlotSharingGroupId()), false, slotProfileForLocation(loc1), TestingUtils.infiniteTime()).get();

	// check that each slot got two
	assertEquals(s1.getTaskManagerLocation(), s3.getTaskManagerLocation());
	assertEquals(s2.getTaskManagerLocation(), s4.getTaskManagerLocation());

	s1.releaseSlot();
	s2.releaseSlot();
	s3.releaseSlot();
	s4.releaseSlot();

	assertEquals(2, testingSlotProvider.getNumberOfAvailableSlots());
}
 
Example #25
Source Project: flink   Author: apache   File: ExecutionVertex.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Gets the location preferences of the vertex's current task execution, as determined by the locations
 * of the predecessors from which it receives input data.
 * If there are more than MAX_DISTINCT_LOCATIONS_TO_CONSIDER different locations of source data, this
 * method returns {@code null} to indicate no location preference.
 *
 * @return The preferred locations based in input streams, or an empty iterable,
 *         if there is no input-based preference.
 */
public Collection<CompletableFuture<TaskManagerLocation>> getPreferredLocationsBasedOnInputs() {
	// otherwise, base the preferred locations on the input connections
	if (inputEdges == null) {
		return Collections.emptySet();
	}
	else {
		Set<CompletableFuture<TaskManagerLocation>> locations = new HashSet<>(getTotalNumberOfParallelSubtasks());
		Set<CompletableFuture<TaskManagerLocation>> inputLocations = new HashSet<>(getTotalNumberOfParallelSubtasks());

		// go over all inputs
		for (int i = 0; i < inputEdges.length; i++) {
			inputLocations.clear();
			ExecutionEdge[] sources = inputEdges[i];
			if (sources != null) {
				// go over all input sources
				for (int k = 0; k < sources.length; k++) {
					// look-up assigned slot of input source
					CompletableFuture<TaskManagerLocation> locationFuture = sources[k].getSource().getProducer().getCurrentTaskManagerLocationFuture();
					// add input location
					inputLocations.add(locationFuture);
					// inputs which have too many distinct sources are not considered
					if (inputLocations.size() > MAX_DISTINCT_LOCATIONS_TO_CONSIDER) {
						inputLocations.clear();
						break;
					}
				}
			}
			// keep the locations of the input with the least preferred locations
			if (locations.isEmpty() || // nothing assigned yet
					(!inputLocations.isEmpty() && inputLocations.size() < locations.size())) {
				// current input has fewer preferred locations
				locations.clear();
				locations.addAll(inputLocations);
			}
		}

		return locations.isEmpty() ? Collections.emptyList() : locations;
	}
}
 
Example #26
Source Project: Flink-CEPplus   Author: ljygz   File: SchedulerTestUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static Execution getTestVertex(Instance... preferredInstances) {
	List<TaskManagerLocation> locations = new ArrayList<>(preferredInstances.length);
	for (Instance i : preferredInstances) {
		locations.add(i.getTaskManagerLocation());
	}
	return getTestVertex(locations);
}
 
Example #27
Source Project: Flink-CEPplus   Author: ljygz   File: SubtaskExecutionAttemptDetailsInfo.java    License: Apache License 2.0 5 votes vote down vote up
public static SubtaskExecutionAttemptDetailsInfo create(AccessExecution execution, MutableIOMetrics ioMetrics) {
	final ExecutionState status = execution.getState();
	final long now = System.currentTimeMillis();

	final TaskManagerLocation location = execution.getAssignedResourceLocation();
	final String locationString = location == null ? "(unassigned)" : location.getHostname();

	long startTime = execution.getStateTimestamp(ExecutionState.DEPLOYING);
	if (startTime == 0) {
		startTime = -1;
	}
	final long endTime = status.isTerminal() ? execution.getStateTimestamp(status) : -1;
	final long duration = startTime > 0 ? ((endTime > 0 ? endTime : now) - startTime) : -1;

	final IOMetricsInfo ioMetricsInfo = new IOMetricsInfo(
		ioMetrics.getNumBytesInLocal() + ioMetrics.getNumBytesInRemote(),
		ioMetrics.isNumBytesInLocalComplete() && ioMetrics.isNumBytesInRemoteComplete(),
		ioMetrics.getNumBytesOut(),
		ioMetrics.isNumBytesOutComplete(),
		ioMetrics.getNumRecordsIn(),
		ioMetrics.isNumRecordsInComplete(),
		ioMetrics.getNumRecordsOut(),
		ioMetrics.isNumRecordsOutComplete());

	return new SubtaskExecutionAttemptDetailsInfo(
		execution.getParallelSubtaskIndex(),
		status,
		execution.getAttemptNumber(),
		locationString,
		startTime,
		endTime,
		duration,
		ioMetricsInfo
	);
}
 
Example #28
Source Project: flink   Author: apache   File: ExecutionVertexDeploymentTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the lazy scheduling flag is correctly forwarded to the produced partition descriptors.
 */
@Test
public void testTddProducedPartitionsLazyScheduling() throws Exception {
	for (ScheduleMode scheduleMode: ScheduleMode.values()) {
		ExecutionJobVertex jobVertex = ExecutionGraphTestUtils.getExecutionJobVertex(
			new JobVertexID(),
			new DirectScheduledExecutorService(),
			scheduleMode);

		IntermediateResult result =
			new IntermediateResult(new IntermediateDataSetID(), jobVertex, 1, ResultPartitionType.PIPELINED);

		ExecutionAttemptID attemptID = new ExecutionAttemptID();
		ExecutionVertex vertex =
			new ExecutionVertex(jobVertex, 0, new IntermediateResult[]{result}, Time.minutes(1));
		TaskDeploymentDescriptorFactory tddFactory =
			TaskDeploymentDescriptorFactory.fromExecutionVertex(vertex, 1);

		ExecutionEdge mockEdge = createMockExecutionEdge(1);

		result.getPartitions()[0].addConsumerGroup();
		result.getPartitions()[0].addConsumer(mockEdge, 0);

		TaskManagerLocation location =
			new TaskManagerLocation(ResourceID.generate(), InetAddress.getLoopbackAddress(), 1);

		TaskDeploymentDescriptor tdd = tddFactory.createDeploymentDescriptor(
			new AllocationID(),
			0,
			null,
			Execution.registerProducedPartitions(vertex, location, attemptID, scheduleMode.allowLazyDeployment()).get().values());

		Collection<ResultPartitionDeploymentDescriptor> producedPartitions = tdd.getProducedPartitions();

		assertEquals(1, producedPartitions.size());
		ResultPartitionDeploymentDescriptor desc = producedPartitions.iterator().next();
		assertEquals(scheduleMode.allowLazyDeployment(), desc.sendScheduleOrUpdateConsumersMessage());
	}
}
 
Example #29
Source Project: flink   Author: flink-tpc-ds   File: SlotProfile.java    License: Apache License 2.0 5 votes vote down vote up
public SlotProfile(
	@Nonnull ResourceProfile resourceProfile,
	@Nonnull Collection<TaskManagerLocation> preferredLocations,
	@Nonnull Collection<AllocationID> preferredAllocations) {

	this(resourceProfile, preferredLocations, preferredAllocations, Collections.emptySet());
}
 
Example #30
Source Project: flink   Author: flink-tpc-ds   File: ArchivedExecutionBuilder.java    License: Apache License 2.0 5 votes vote down vote up
public ArchivedExecution build() throws UnknownHostException {
	return new ArchivedExecution(
		userAccumulators != null ? userAccumulators : new StringifiedAccumulatorResult[0],
		ioMetrics != null ? ioMetrics : new TestIOMetrics(),
		attemptId != null ? attemptId : new ExecutionAttemptID(),
		attemptNumber,
		state != null ? state : ExecutionState.FINISHED,
		failureCause != null ? failureCause : "(null)",
		assignedResourceLocation != null ? assignedResourceLocation : new TaskManagerLocation(new ResourceID("tm"), InetAddress.getLocalHost(), 1234),
		assignedAllocationID != null ? assignedAllocationID : new AllocationID(0L, 0L),
		parallelSubtaskIndex,
		stateTimestamps != null ? stateTimestamps : new long[]{1, 2, 3, 4, 5, 5, 5, 5}
	);
}