org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID Java Examples

The following examples show how to use org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ExecutionGraph.java    From flink with Apache License 2.0 6 votes vote down vote up
ResultPartitionID createResultPartitionId(final IntermediateResultPartitionID resultPartitionId) {
	final SchedulingResultPartition schedulingResultPartition = schedulingTopology.getResultPartitionOrThrow(resultPartitionId);
	final SchedulingExecutionVertex producer = schedulingResultPartition.getProducer();
	final ExecutionVertexID producerId = producer.getId();
	final JobVertexID jobVertexId = producerId.getJobVertexId();
	final ExecutionJobVertex jobVertex = getJobVertex(jobVertexId);
	checkNotNull(jobVertex, "Unknown job vertex %s", jobVertexId);

	final ExecutionVertex[] taskVertices = jobVertex.getTaskVertices();
	final int subtaskIndex = producerId.getSubtaskIndex();
	checkState(subtaskIndex < taskVertices.length, "Invalid subtask index %d for job vertex %s", subtaskIndex, jobVertexId);

	final ExecutionVertex taskVertex = taskVertices[subtaskIndex];
	final Execution execution = taskVertex.getCurrentExecutionAttempt();
	return new ResultPartitionID(resultPartitionId, execution.getAttemptId());
}
 
Example #2
Source File: ExecutionGraph.java    From flink with Apache License 2.0 6 votes vote down vote up
ResultPartitionID createResultPartitionId(final IntermediateResultPartitionID resultPartitionId) {
	final SchedulingResultPartition schedulingResultPartition =
		getSchedulingTopology().getResultPartition(resultPartitionId);
	final SchedulingExecutionVertex producer = schedulingResultPartition.getProducer();
	final ExecutionVertexID producerId = producer.getId();
	final JobVertexID jobVertexId = producerId.getJobVertexId();
	final ExecutionJobVertex jobVertex = getJobVertex(jobVertexId);
	checkNotNull(jobVertex, "Unknown job vertex %s", jobVertexId);

	final ExecutionVertex[] taskVertices = jobVertex.getTaskVertices();
	final int subtaskIndex = producerId.getSubtaskIndex();
	checkState(subtaskIndex < taskVertices.length, "Invalid subtask index %d for job vertex %s", subtaskIndex, jobVertexId);

	final ExecutionVertex taskVertex = taskVertices[subtaskIndex];
	final Execution execution = taskVertex.getCurrentExecutionAttempt();
	return new ResultPartitionID(resultPartitionId, execution.getAttemptId());
}
 
Example #3
Source File: SingleInputGateTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 120 * 1000)
public void testIsMoreAvailableReadingFromSingleInputChannel() throws Exception {
	// Setup
	final SingleInputGate inputGate = createInputGate();

	final TestInputChannel[] inputChannels = new TestInputChannel[]{
		new TestInputChannel(inputGate, 0),
		new TestInputChannel(inputGate, 1)
	};

	inputGate.setInputChannel(
		new IntermediateResultPartitionID(), inputChannels[0]);

	inputGate.setInputChannel(
		new IntermediateResultPartitionID(), inputChannels[1]);

	// Test
	inputChannels[0].readBuffer();
	inputChannels[0].readBuffer(false);

	inputGate.notifyChannelNonEmpty(inputChannels[0]);

	verifyBufferOrEvent(inputGate, true, 0, true);
	verifyBufferOrEvent(inputGate, true, 0, false);
}
 
Example #4
Source File: ResultPartitionManager.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public void releasePartitionsProducedBy(ExecutionAttemptID executionId, Throwable cause) {
	synchronized (registeredPartitions) {
		final Map<IntermediateResultPartitionID, ResultPartition> partitions =
				registeredPartitions.row(executionId);

		for (ResultPartition partition : partitions.values()) {
			partition.release(cause);
		}

		for (IntermediateResultPartitionID partitionId : ImmutableList
				.copyOf(partitions.keySet())) {

			registeredPartitions.remove(executionId, partitionId);
		}

		LOG.debug("Released all partitions produced by {}.", executionId);
	}
}
 
Example #5
Source File: RegionPartitionReleaseStrategyTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void releasePartitionsIfDownstreamRegionWithMultipleOperatorsIsFinished() {
	final List<TestingSchedulingExecutionVertex> sourceVertices = testingSchedulingTopology.addExecutionVertices().finish();
	final List<TestingSchedulingExecutionVertex> intermediateVertices = testingSchedulingTopology.addExecutionVertices().finish();
	final List<TestingSchedulingExecutionVertex> sinkVertices = testingSchedulingTopology.addExecutionVertices().finish();
	final List<TestingSchedulingResultPartition> sourceResultPartitions = testingSchedulingTopology.connectAllToAll(sourceVertices, intermediateVertices).finish();
	testingSchedulingTopology.connectAllToAll(intermediateVertices, sinkVertices).withResultPartitionType(ResultPartitionType.PIPELINED).finish();

	final ExecutionVertexID onlyIntermediateVertexId = intermediateVertices.get(0).getId();
	final ExecutionVertexID onlySinkVertexId = sinkVertices.get(0).getId();
	final IntermediateResultPartitionID onlySourceResultPartitionId = sourceResultPartitions.get(0).getId();

	final RegionPartitionReleaseStrategy regionPartitionReleaseStrategy = new RegionPartitionReleaseStrategy(testingSchedulingTopology);

	regionPartitionReleaseStrategy.vertexFinished(onlyIntermediateVertexId);
	final List<IntermediateResultPartitionID> partitionsToRelease = regionPartitionReleaseStrategy.vertexFinished(onlySinkVertexId);
	assertThat(partitionsToRelease, contains(onlySourceResultPartitionId));
}
 
Example #6
Source File: NettyMessage.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
static TaskEventRequest readFrom(ByteBuf buffer, ClassLoader classLoader) throws IOException {
	// directly deserialize fromNetty's buffer
	int length = buffer.readInt();
	ByteBuffer serializedEvent = buffer.nioBuffer(buffer.readerIndex(), length);
	// assume this event's content is read from the ByteBuf (positions are not shared!)
	buffer.readerIndex(buffer.readerIndex() + length);

	TaskEvent event =
		(TaskEvent) EventSerializer.fromSerializedEvent(serializedEvent, classLoader);

	ResultPartitionID partitionId =
		new ResultPartitionID(
			IntermediateResultPartitionID.fromByteBuf(buffer),
			ExecutionAttemptID.fromByteBuf(buffer));

	InputChannelID receiverId = InputChannelID.fromByteBuf(buffer);

	return new TaskEventRequest(event, partitionId, receiverId);
}
 
Example #7
Source File: ResultPartitionDeploymentDescriptor.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public ResultPartitionDeploymentDescriptor(
		IntermediateDataSetID resultId,
		IntermediateResultPartitionID partitionId,
		ResultPartitionType partitionType,
		int numberOfSubpartitions,
		int maxParallelism,
		boolean lazyScheduling) {

	this.resultId = checkNotNull(resultId);
	this.partitionId = checkNotNull(partitionId);
	this.partitionType = checkNotNull(partitionType);

	KeyGroupRangeAssignment.checkParallelismPreconditions(maxParallelism);
	checkArgument(numberOfSubpartitions >= 1);
	this.numberOfSubpartitions = numberOfSubpartitions;
	this.maxParallelism = maxParallelism;
	this.sendScheduleOrUpdateConsumersMessage = lazyScheduling;
}
 
Example #8
Source File: RegionPartitionReleaseStrategyTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void toggleVertexFinishedUnfinished() {
	final List<TestingSchedulingExecutionVertex> producers = testingSchedulingTopology.addExecutionVertices().finish();
	final List<TestingSchedulingExecutionVertex> consumers = testingSchedulingTopology.addExecutionVertices().withParallelism(2).finish();
	testingSchedulingTopology.connectAllToAll(producers, consumers).finish();

	final ExecutionVertexID consumerVertex1 = consumers.get(0).getId();
	final ExecutionVertexID consumerVertex2 = consumers.get(1).getId();

	final RegionPartitionReleaseStrategy regionPartitionReleaseStrategy = new RegionPartitionReleaseStrategy(testingSchedulingTopology);

	regionPartitionReleaseStrategy.vertexFinished(consumerVertex1);
	regionPartitionReleaseStrategy.vertexFinished(consumerVertex2);
	regionPartitionReleaseStrategy.vertexUnfinished(consumerVertex2);

	final List<IntermediateResultPartitionID> partitionsToRelease = regionPartitionReleaseStrategy.vertexFinished(consumerVertex1);
	assertThat(partitionsToRelease, is(empty()));
}
 
Example #9
Source File: NettyMessage.java    From flink with Apache License 2.0 6 votes vote down vote up
static TaskEventRequest readFrom(ByteBuf buffer, ClassLoader classLoader) throws IOException {
	// directly deserialize fromNetty's buffer
	int length = buffer.readInt();
	ByteBuffer serializedEvent = buffer.nioBuffer(buffer.readerIndex(), length);
	// assume this event's content is read from the ByteBuf (positions are not shared!)
	buffer.readerIndex(buffer.readerIndex() + length);

	TaskEvent event =
		(TaskEvent) EventSerializer.fromSerializedEvent(serializedEvent, classLoader);

	ResultPartitionID partitionId =
		new ResultPartitionID(
			IntermediateResultPartitionID.fromByteBuf(buffer),
			ExecutionAttemptID.fromByteBuf(buffer));

	InputChannelID receiverId = InputChannelID.fromByteBuf(buffer);

	return new TaskEventRequest(event, partitionId, receiverId);
}
 
Example #10
Source File: RegionPartitionReleaseStrategy.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public List<IntermediateResultPartitionID> vertexFinished(final ExecutionVertexID finishedVertex) {
	final PipelinedRegionExecutionView regionExecutionView = getPipelinedRegionExecutionViewForVertex(finishedVertex);
	regionExecutionView.vertexFinished(finishedVertex);

	if (regionExecutionView.isFinished()) {
		final PipelinedRegion pipelinedRegion = getPipelinedRegionForVertex(finishedVertex);
		final PipelinedRegionConsumedBlockingPartitions consumedPartitionsOfVertexRegion = getConsumedBlockingPartitionsForRegion(pipelinedRegion);
		return filterReleasablePartitions(consumedPartitionsOfVertexRegion);
	}
	return Collections.emptyList();
}
 
Example #11
Source File: DefaultExecutionTopologyTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testResultPartitionOrThrow() {
	try {
		adapter.getResultPartition(new IntermediateResultPartitionID());
		fail("get not exist result partition");
	} catch (IllegalArgumentException exception) {
		// expected
	}
}
 
Example #12
Source File: RegionPartitionReleaseStrategy.java    From flink with Apache License 2.0 5 votes vote down vote up
private List<IntermediateResultPartitionID> filterReleasablePartitions(final PipelinedRegionConsumedBlockingPartitions consumedPartitionsOfVertexRegion) {
	return consumedPartitionsOfVertexRegion
		.getConsumedBlockingPartitions()
		.stream()
		.filter(this::areConsumerRegionsFinished)
		.collect(Collectors.toList());
}
 
Example #13
Source File: TestingSchedulingTopology.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public TestingSchedulingResultPartition getResultPartition(final IntermediateResultPartitionID intermediateResultPartitionId) {
	final TestingSchedulingResultPartition resultPartition = schedulingResultPartitions.get(intermediateResultPartitionId);
	if (resultPartition == null) {
		throw new IllegalArgumentException("can not find partition: " + intermediateResultPartitionId);
	}
	return resultPartition;
}
 
Example #14
Source File: ExecutionGraphResultPartitionAvailabilityCheckerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testPartitionAvailabilityCheck() {

	final IntermediateResultPartitionID irp1ID = new IntermediateResultPartitionID();
	final IntermediateResultPartitionID irp2ID = new IntermediateResultPartitionID();
	final IntermediateResultPartitionID irp3ID = new IntermediateResultPartitionID();
	final IntermediateResultPartitionID irp4ID = new IntermediateResultPartitionID();

	final Map<IntermediateResultPartitionID, Boolean> expectedAvailability =
		new HashMap<IntermediateResultPartitionID, Boolean>() {{
			put(irp1ID, true);
			put(irp2ID, false);
			put(irp3ID, false);
			put(irp4ID, true);
		}};

	// let the partition tracker respect the expected availability result
	final TestingJobMasterPartitionTracker partitionTracker = new TestingJobMasterPartitionTracker();
	partitionTracker.setIsPartitionTrackedFunction(rpID -> expectedAvailability.get(rpID.getPartitionId()));

	// the execution attempt ID should make no difference in this case
	final Function<IntermediateResultPartitionID, ResultPartitionID> partitionIDMapper =
		intermediateResultPartitionID -> new ResultPartitionID(intermediateResultPartitionID, new ExecutionAttemptID());

	final ResultPartitionAvailabilityChecker resultPartitionAvailabilityChecker =
		new ExecutionGraphResultPartitionAvailabilityChecker(partitionIDMapper, partitionTracker);

	for (IntermediateResultPartitionID irpID : expectedAvailability.keySet()) {
		assertEquals(expectedAvailability.get(irpID), resultPartitionAvailabilityChecker.isAvailable(irpID));
	}
}
 
Example #15
Source File: SingleInputGateTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private static Map<InputGateID, SingleInputGate> createInputGateWithLocalChannels(
		NettyShuffleEnvironment network,
		int numberOfGates,
		@SuppressWarnings("SameParameterValue") int numberOfLocalChannels) {
	ShuffleDescriptor[] channelDescs = new NettyShuffleDescriptor[numberOfLocalChannels];
	for (int i = 0; i < numberOfLocalChannels; i++) {
		channelDescs[i] = createRemoteWithIdAndLocation(new IntermediateResultPartitionID(), ResourceID.generate());
	}

	InputGateDeploymentDescriptor[] gateDescs = new InputGateDeploymentDescriptor[numberOfGates];
	IntermediateDataSetID[] ids = new IntermediateDataSetID[numberOfGates];
	for (int i = 0; i < numberOfGates; i++) {
		ids[i] = new IntermediateDataSetID();
		gateDescs[i] = new InputGateDeploymentDescriptor(
			ids[i],
			ResultPartitionType.PIPELINED,
			0,
			channelDescs);
	}

	ExecutionAttemptID consumerID = new ExecutionAttemptID();
	SingleInputGate[] gates = network.createInputGates(
		network.createShuffleIOOwnerContext("", consumerID, new UnregisteredMetricsGroup()),
		SingleInputGateBuilder.NO_OP_PRODUCER_CHECKER,
		Arrays.asList(gateDescs)).toArray(new SingleInputGate[] {});
	Map<InputGateID, SingleInputGate> inputGatesById = new HashMap<>();
	for (int i = 0; i < numberOfGates; i++) {
		inputGatesById.put(new InputGateID(ids[i], consumerID), gates[i]);
	}

	return inputGatesById;
}
 
Example #16
Source File: IteratorWrappingTestSingleInputGate.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private IteratorWrappingTestSingleInputGate<T> wrapIterator(MutableObjectIterator<T> iterator) throws IOException, InterruptedException {
	inputIterator = iterator;
	serializer = new SpanningRecordSerializer<T>();

	// The input iterator can produce an infinite stream. That's why we have to serialize each
	// record on demand and cannot do it upfront.
	final BufferAndAvailabilityProvider answer = new BufferAndAvailabilityProvider() {

		private boolean hasData = inputIterator.next(reuse) != null;

		@Override
		public Optional<BufferAndAvailability> getBufferAvailability() throws IOException {
			if (hasData) {
				serializer.serializeRecord(reuse);
				BufferBuilder bufferBuilder = createBufferBuilder(bufferSize);
				serializer.copyToBufferBuilder(bufferBuilder);

				hasData = inputIterator.next(reuse) != null;

				// Call getCurrentBuffer to ensure size is set
				return Optional.of(new BufferAndAvailability(buildSingleBuffer(bufferBuilder), true, 0));
			} else {
				inputChannel.setReleased();

				return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE),
					false,
					0));
			}
		}
	};

	inputChannel.addBufferAndAvailability(answer);

	inputGate.setInputChannel(new IntermediateResultPartitionID(), inputChannel);

	return this;
}
 
Example #17
Source File: ResultPartitionDeploymentDescriptorTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests simple de/serialization.
 */
@Test
public void testSerialization() throws Exception {
	// Expected values
	IntermediateDataSetID resultId = new IntermediateDataSetID();
	IntermediateResultPartitionID partitionId = new IntermediateResultPartitionID();
	ResultPartitionType partitionType = ResultPartitionType.PIPELINED;
	int numberOfSubpartitions = 24;

	ResultPartitionDeploymentDescriptor orig =
			new ResultPartitionDeploymentDescriptor(
					resultId,
					partitionId,
					partitionType,
					numberOfSubpartitions,
					numberOfSubpartitions,
					true);

	ResultPartitionDeploymentDescriptor copy =
			CommonTestUtils.createCopySerializable(orig);

	assertEquals(resultId, copy.getResultId());
	assertEquals(partitionId, copy.getPartitionId());
	assertEquals(partitionType, copy.getPartitionType());
	assertEquals(numberOfSubpartitions, copy.getNumberOfSubpartitions());
	assertTrue(copy.sendScheduleOrUpdateConsumersMessage());
}
 
Example #18
Source File: RegionPartitionReleaseStrategyTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void notReleasePartitionsIfDownstreamRegionIsNotFinished() {
	final List<TestingSchedulingExecutionVertex> producers = testingSchedulingTopology.addExecutionVertices().finish();
	final List<TestingSchedulingExecutionVertex> consumers = testingSchedulingTopology.addExecutionVertices().withParallelism(2).finish();
	testingSchedulingTopology.connectAllToAll(producers, consumers).finish();

	final ExecutionVertexID consumerVertex1 = consumers.get(0).getId();

	final RegionPartitionReleaseStrategy regionPartitionReleaseStrategy = new RegionPartitionReleaseStrategy(testingSchedulingTopology);

	final List<IntermediateResultPartitionID> partitionsToRelease = regionPartitionReleaseStrategy.vertexFinished(consumerVertex1);
	assertThat(partitionsToRelease, is(empty()));
}
 
Example #19
Source File: IntermediateResultPartition.java    From flink with Apache License 2.0 5 votes vote down vote up
public IntermediateResultPartition(IntermediateResult totalResult, ExecutionVertex producer, int partitionNumber) {
	this.totalResult = totalResult;
	this.producer = producer;
	this.partitionNumber = partitionNumber;
	this.consumers = new ArrayList<List<ExecutionEdge>>(0);
	this.partitionId = new IntermediateResultPartitionID(totalResult.getId(), partitionNumber);
}
 
Example #20
Source File: SingleInputGateTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private static Map<InputGateID, SingleInputGate> createInputGateWithLocalChannels(
		NettyShuffleEnvironment network,
		int numberOfGates,
		@SuppressWarnings("SameParameterValue") int numberOfLocalChannels) {
	ShuffleDescriptor[] channelDescs = new NettyShuffleDescriptor[numberOfLocalChannels];
	for (int i = 0; i < numberOfLocalChannels; i++) {
		channelDescs[i] = createRemoteWithIdAndLocation(new IntermediateResultPartitionID(), ResourceID.generate());
	}

	InputGateDeploymentDescriptor[] gateDescs = new InputGateDeploymentDescriptor[numberOfGates];
	IntermediateDataSetID[] ids = new IntermediateDataSetID[numberOfGates];
	for (int i = 0; i < numberOfGates; i++) {
		ids[i] = new IntermediateDataSetID();
		gateDescs[i] = new InputGateDeploymentDescriptor(
			ids[i],
			ResultPartitionType.PIPELINED,
			0,
			channelDescs);
	}

	ExecutionAttemptID consumerID = new ExecutionAttemptID();
	SingleInputGate[] gates = network.createInputGates(
		network.createShuffleIOOwnerContext("", consumerID, new UnregisteredMetricsGroup()),
		SingleInputGateBuilder.NO_OP_PRODUCER_CHECKER,
		asList(gateDescs)).toArray(new SingleInputGate[] {});
	Map<InputGateID, SingleInputGate> inputGatesById = new HashMap<>();
	for (int i = 0; i < numberOfGates; i++) {
		inputGatesById.put(new InputGateID(ids[i], consumerID), gates[i]);
	}

	return inputGatesById;
}
 
Example #21
Source File: ExecutionGraphToSchedulingTopologyAdapter.java    From flink with Apache License 2.0 5 votes vote down vote up
private static List<DefaultSchedulingResultPartition> generateProducedSchedulingResultPartition(
	Map<IntermediateResultPartitionID, IntermediateResultPartition> producedIntermediatePartitions) {

	List<DefaultSchedulingResultPartition> producedSchedulingPartitions = new ArrayList<>(producedIntermediatePartitions.size());

	producedIntermediatePartitions.values().forEach(
		irp -> producedSchedulingPartitions.add(
			new DefaultSchedulingResultPartition(
				irp.getPartitionId(),
				irp.getIntermediateResult().getId(),
				irp.getResultType())));

	return producedSchedulingPartitions;
}
 
Example #22
Source File: DefaultExecutionTopology.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public DefaultResultPartition getResultPartition(final IntermediateResultPartitionID intermediateResultPartitionId) {
	final DefaultResultPartition resultPartition = resultPartitionsById.get(intermediateResultPartitionId);
	if (resultPartition == null) {
		throw new IllegalArgumentException("can not find partition: " + intermediateResultPartitionId);
	}
	return resultPartition;
}
 
Example #23
Source File: ExecutionGraphResultPartitionAvailabilityChecker.java    From flink with Apache License 2.0 5 votes vote down vote up
ExecutionGraphResultPartitionAvailabilityChecker(
		final Function<IntermediateResultPartitionID, ResultPartitionID> partitionIDMapper,
		final PartitionTracker partitionTracker) {

	this.partitionIDMapper = checkNotNull(partitionIDMapper);
	this.partitionTracker = checkNotNull(partitionTracker);
}
 
Example #24
Source File: NettyShuffleDescriptorBuilder.java    From flink with Apache License 2.0 5 votes vote down vote up
public static NettyShuffleDescriptor createRemoteWithIdAndLocation(
		IntermediateResultPartitionID partitionId,
		ResourceID producerLocation) {
	return newBuilder()
		.setId(new ResultPartitionID(partitionId, new ExecutionAttemptID()))
		.setProducerLocation(producerLocation)
		.buildRemote();
}
 
Example #25
Source File: Execution.java    From flink with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
static CompletableFuture<Map<IntermediateResultPartitionID, ResultPartitionDeploymentDescriptor>> registerProducedPartitions(
		ExecutionVertex vertex,
		TaskManagerLocation location,
		ExecutionAttemptID attemptId) {
	ProducerDescriptor producerDescriptor = ProducerDescriptor.create(location, attemptId);

	boolean lazyScheduling = vertex.getExecutionGraph().getScheduleMode().allowLazyDeployment();

	Collection<IntermediateResultPartition> partitions = vertex.getProducedPartitions().values();
	Collection<CompletableFuture<ResultPartitionDeploymentDescriptor>> partitionRegistrations =
		new ArrayList<>(partitions.size());

	for (IntermediateResultPartition partition : partitions) {
		PartitionDescriptor partitionDescriptor = PartitionDescriptor.from(partition);
		int maxParallelism = getPartitionMaxParallelism(partition);
		CompletableFuture<? extends ShuffleDescriptor> shuffleDescriptorFuture = vertex
			.getExecutionGraph()
			.getShuffleMaster()
			.registerPartitionWithProducer(partitionDescriptor, producerDescriptor);

		CompletableFuture<ResultPartitionDeploymentDescriptor> partitionRegistration = shuffleDescriptorFuture
			.thenApply(shuffleDescriptor -> new ResultPartitionDeploymentDescriptor(
				partitionDescriptor,
				shuffleDescriptor,
				maxParallelism,
				lazyScheduling));
		partitionRegistrations.add(partitionRegistration);
	}

	return FutureUtils.combineAll(partitionRegistrations).thenApply(rpdds -> {
		Map<IntermediateResultPartitionID, ResultPartitionDeploymentDescriptor> producedPartitions =
			new LinkedHashMap<>(partitions.size());
		rpdds.forEach(rpdd -> producedPartitions.put(rpdd.getPartitionId(), rpdd));
		return producedPartitions;
	});
}
 
Example #26
Source File: DefaultExecutionVertexTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetConsumedResultPartitions() {
	IntermediateResultPartitionID partitionIds1 = IterableUtils
		.toStream(consumerVertex.getConsumedResults())
		.findAny()
		.map(SchedulingResultPartition::getId)
		.orElseThrow(() -> new IllegalArgumentException("can not find result partition"));
	assertEquals(partitionIds1, intermediateResultPartitionId);
}
 
Example #27
Source File: IntermediateResult.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Returns the partition with the given ID.
 *
 * @param resultPartitionId ID of the partition to look up
 * @throws NullPointerException If partition ID <code>null</code>
 * @throws IllegalArgumentException Thrown if unknown partition ID
 * @return Intermediate result partition with the given ID
 */
public IntermediateResultPartition getPartitionById(IntermediateResultPartitionID resultPartitionId) {
	// Looks ups the partition number via the helper map and returns the
	// partition. Currently, this happens infrequently enough that we could
	// consider removing the map and scanning the partitions on every lookup.
	// The lookup (currently) only happen when the producer of an intermediate
	// result cannot be found via its registered execution.
	Integer partitionNumber = partitionLookupHelper.get(checkNotNull(resultPartitionId, "IntermediateResultPartitionID"));
	if (partitionNumber != null) {
		return partitions[partitionNumber];
	} else {
		throw new IllegalArgumentException("Unknown intermediate result partition ID " + resultPartitionId);
	}
}
 
Example #28
Source File: ExecutionGraph.java    From flink with Apache License 2.0 5 votes vote down vote up
private void maybeReleasePartitions(final Execution attempt) {
	final ExecutionVertexID finishedExecutionVertex = attempt.getVertex().getID();

	if (attempt.getState() == ExecutionState.FINISHED) {
		final List<IntermediateResultPartitionID> releasablePartitions = partitionReleaseStrategy.vertexFinished(finishedExecutionVertex);
		releasePartitions(releasablePartitions);
	} else {
		partitionReleaseStrategy.vertexUnfinished(finishedExecutionVertex);
	}
}
 
Example #29
Source File: TestingSchedulingResultPartition.java    From flink with Apache License 2.0 5 votes vote down vote up
TestingSchedulingResultPartition(IntermediateDataSetID dataSetID, ResultPartitionType type, ResultPartitionState state) {
	this.intermediateDataSetID = dataSetID;
	this.partitionType = type;
	this.state = state;
	this.intermediateResultPartitionID = new IntermediateResultPartitionID();
	this.consumers = new ArrayList<>();
}
 
Example #30
Source File: DefaultSchedulingExecutionVertexTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetConsumedResultPartitions() {
	IntermediateResultPartitionID partitionIds1 = consumerVertex
		.getConsumedResultPartitions().stream().findAny().map(SchedulingResultPartition::getId)
		.orElseThrow(() -> new IllegalArgumentException("can not find result partition"));
	assertEquals(partitionIds1, intermediateResultPartitionId);
}