Java Code Examples for org.apache.flink.runtime.jobgraph.JobVertexID

The following examples show how to use org.apache.flink.runtime.jobgraph.JobVertexID. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: SchedulerTestBase.java    License: Apache License 2.0 6 votes vote down vote up
public int getNumberOfAvailableSlotsForGroup(SlotSharingGroupId slotSharingGroupId, JobVertexID jobVertexId) {
	final SlotSharingManager multiTaskSlotManager = slotSharingManagersMap.get(slotSharingGroupId);

	if (multiTaskSlotManager != null) {
		int availableSlots = 0;

		for (SlotSharingManager.MultiTaskSlot multiTaskSlot : multiTaskSlotManager.getResolvedRootSlots()) {
			if (!multiTaskSlot.contains(jobVertexId)) {
				availableSlots++;
			}
		}

		return availableSlots;
	} else {
		throw new FlinkRuntimeException("No MultiTaskSlotmanager registered under " + slotSharingGroupId + '.');
	}
}
 
Example 2
Source Project: flink   Source File: OperatorGroupTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGenerateScopeDefault() throws Exception {
	TaskManagerMetricGroup tmGroup = new TaskManagerMetricGroup(registry, "theHostName", "test-tm-id");
	TaskManagerJobMetricGroup jmGroup = new TaskManagerJobMetricGroup(registry, tmGroup, new JobID(), "myJobName");
	TaskMetricGroup taskGroup = new TaskMetricGroup(
			registry, jmGroup,  new JobVertexID(),  new AbstractID(), "aTaskName", 11, 0);
	OperatorMetricGroup opGroup = new OperatorMetricGroup(registry, taskGroup, new OperatorID(), "myOpName");

	assertArrayEquals(
			new String[] { "theHostName", "taskmanager", "test-tm-id", "myJobName", "myOpName", "11" },
			opGroup.getScopeComponents());

	assertEquals(
			"theHostName.taskmanager.test-tm-id.myJobName.myOpName.11.name",
			opGroup.getMetricIdentifier("name"));
}
 
Example 3
Source Project: flink   Source File: CheckpointMetadataLoadingTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests correct savepoint loading.
 */
@Test
public void testAllStateRestored() throws Exception {
	final JobID jobId = new JobID();
	final OperatorID operatorId = new OperatorID();
	final long checkpointId = Integer.MAX_VALUE + 123123L;
	final int parallelism = 128128;

	final CompletedCheckpointStorageLocation testSavepoint = createSavepointWithOperatorSubtaskState(checkpointId, operatorId, parallelism);
	final Map<JobVertexID, ExecutionJobVertex> tasks = createTasks(operatorId, parallelism, parallelism);

	final CompletedCheckpoint loaded = Checkpoints.loadAndValidateCheckpoint(jobId, tasks, testSavepoint, cl, false);

	assertEquals(jobId, loaded.getJobId());
	assertEquals(checkpointId, loaded.getCheckpointID());
}
 
Example 4
Source Project: flink   Source File: CheckpointCoordinatorTest.java    License: Apache License 2.0 6 votes vote down vote up
public static ChainedStateHandle<OperatorStateHandle> generateChainedPartitionableStateHandle(
		JobVertexID jobVertexID,
		int index,
		int namedStates,
		int partitionsPerState,
		boolean rawState) throws IOException {

	Map<String, List<? extends Serializable>> statesListsMap = new HashMap<>(namedStates);

	for (int i = 0; i < namedStates; ++i) {
		List<Integer> testStatesLists = new ArrayList<>(partitionsPerState);
		// generate state
		int seed = jobVertexID.hashCode() * index + i * namedStates;
		if (rawState) {
			seed = (seed + 1) * 31;
		}
		Random random = new Random(seed);
		for (int j = 0; j < partitionsPerState; ++j) {
			int simulatedStateValue = random.nextInt();
			testStatesLists.add(simulatedStateValue);
		}
		statesListsMap.put("state-" + i, testStatesLists);
	}

	return ChainedStateHandle.wrapSingleHandle(generatePartitionableStateHandle(statesListsMap));
}
 
Example 5
Source Project: Flink-CEPplus   Source File: CheckpointCoordinatorTest.java    License: Apache License 2.0 6 votes vote down vote up
static TaskStateSnapshot mockSubtaskState(
	JobVertexID jobVertexID,
	int index,
	KeyGroupRange keyGroupRange) throws IOException {

	OperatorStateHandle partitionableState = generatePartitionableStateHandle(jobVertexID, index, 2, 8, false);
	KeyGroupsStateHandle partitionedKeyGroupState = generateKeyGroupState(jobVertexID, keyGroupRange, false);

	TaskStateSnapshot subtaskStates = spy(new TaskStateSnapshot());
	OperatorSubtaskState subtaskState = spy(new OperatorSubtaskState(
		partitionableState, null, partitionedKeyGroupState, null)
	);

	subtaskStates.putSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID), subtaskState);

	return subtaskStates;
}
 
Example 6
Source Project: Flink-CEPplus   Source File: CheckpointStatistics.java    License: Apache License 2.0 6 votes vote down vote up
@JsonCreator
public PendingCheckpointStatistics(
	@JsonProperty(FIELD_NAME_ID) long id,
	@JsonProperty(FIELD_NAME_STATUS) CheckpointStatsStatus status,
	@JsonProperty(FIELD_NAME_IS_SAVEPOINT) boolean savepoint,
	@JsonProperty(FIELD_NAME_TRIGGER_TIMESTAMP) long triggerTimestamp,
	@JsonProperty(FIELD_NAME_LATEST_ACK_TIMESTAMP) long latestAckTimestamp,
	@JsonProperty(FIELD_NAME_STATE_SIZE) long stateSize,
	@JsonProperty(FIELD_NAME_DURATION) long duration,
	@JsonProperty(FIELD_NAME_ALIGNMENT_BUFFERED) long alignmentBuffered,
	@JsonProperty(FIELD_NAME_NUM_SUBTASKS) int numSubtasks,
	@JsonProperty(FIELD_NAME_NUM_ACK_SUBTASKS) int numAckSubtasks,
	@JsonDeserialize(keyUsing = JobVertexIDKeyDeserializer.class) @JsonProperty(FIELD_NAME_TASKS) Map<JobVertexID, TaskCheckpointStatistics> checkpointingStatisticsPerTask) {
	super(
		id,
		status,
		savepoint,
		triggerTimestamp,
		latestAckTimestamp,
		stateSize,
		duration,
		alignmentBuffered,
		numSubtasks,
		numAckSubtasks,
		checkpointingStatisticsPerTask);
}
 
Example 7
Source Project: Flink-CEPplus   Source File: CheckpointCoordinatorTest.java    License: Apache License 2.0 6 votes vote down vote up
public static OperatorStateHandle generatePartitionableStateHandle(
	JobVertexID jobVertexID,
	int index,
	int namedStates,
	int partitionsPerState,
	boolean rawState) throws IOException {

	Map<String, List<? extends Serializable>> statesListsMap = new HashMap<>(namedStates);

	for (int i = 0; i < namedStates; ++i) {
		List<Integer> testStatesLists = new ArrayList<>(partitionsPerState);
		// generate state
		int seed = jobVertexID.hashCode() * index + i * namedStates;
		if (rawState) {
			seed = (seed + 1) * 31;
		}
		Random random = new Random(seed);
		for (int j = 0; j < partitionsPerState; ++j) {
			int simulatedStateValue = random.nextInt();
			testStatesLists.add(simulatedStateValue);
		}
		statesListsMap.put("state-" + i, testStatesLists);
	}

	return generatePartitionableStateHandle(statesListsMap);
}
 
Example 8
Source Project: flink   Source File: SchedulerTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static Execution getTestVertex(Collection<CompletableFuture<TaskManagerLocation>> preferredLocationFutures) {
	ExecutionJobVertex executionJobVertex = mock(ExecutionJobVertex.class);
	ExecutionVertex vertex = mock(ExecutionVertex.class);

	when(vertex.getPreferredLocationsBasedOnInputs()).thenReturn(preferredLocationFutures);
	when(vertex.getPreferredLocations()).thenReturn(preferredLocationFutures);
	when(vertex.getJobId()).thenReturn(new JobID());
	when(vertex.toString()).thenReturn("TEST-VERTEX");
	when(vertex.getJobVertex()).thenReturn(executionJobVertex);
	when(vertex.getJobvertexId()).thenReturn(new JobVertexID());

	Execution execution = mock(Execution.class);
	when(execution.getVertex()).thenReturn(vertex);
	when(execution.calculatePreferredLocations(any(LocationPreferenceConstraint.class))).thenCallRealMethod();

	return execution;
}
 
Example 9
Source Project: flink   Source File: AbstractExecutionSlotAllocatorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testComputeAllPriorAllocationIds() {
	final List<AllocationID> expectAllocationIds = Arrays.asList(new AllocationID(), new AllocationID());
	final List<ExecutionVertexSchedulingRequirements> testSchedulingRequirements = Arrays.asList(
		new ExecutionVertexSchedulingRequirements.Builder().
			withExecutionVertexId(new ExecutionVertexID(new JobVertexID(), 0)).
			withPreviousAllocationId(expectAllocationIds.get(0)).
			build(),
		new ExecutionVertexSchedulingRequirements.Builder().
			withExecutionVertexId(new ExecutionVertexID(new JobVertexID(), 1)).
			withPreviousAllocationId(expectAllocationIds.get(0)).
			build(),
		new ExecutionVertexSchedulingRequirements.Builder().
			withExecutionVertexId(new ExecutionVertexID(new JobVertexID(), 2)).
			withPreviousAllocationId(expectAllocationIds.get(1)).
			build(),
		new ExecutionVertexSchedulingRequirements.Builder().
			withExecutionVertexId(new ExecutionVertexID(new JobVertexID(), 3)).
			build()
	);

	final Set<AllocationID> allPriorAllocationIds =
		AbstractExecutionSlotAllocator.computeAllPriorAllocationIds(testSchedulingRequirements);
	assertThat(allPriorAllocationIds, containsInAnyOrder(expectAllocationIds.toArray()));
}
 
Example 10
Source Project: Flink-CEPplus   Source File: MetricGroupTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateQueryServiceMetricInfo() {
	JobID jid = new JobID();
	JobVertexID vid = new JobVertexID();
	AbstractID eid = new AbstractID();
	MetricRegistryImpl registry = new MetricRegistryImpl(defaultMetricRegistryConfiguration);
	TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id");
	TaskManagerJobMetricGroup job = new TaskManagerJobMetricGroup(registry, tm, jid, "jobname");
	TaskMetricGroup task = new TaskMetricGroup(registry, job, vid, eid, "taskName", 4, 5);
	GenericMetricGroup userGroup1 = new GenericMetricGroup(registry, task, "hello");
	GenericMetricGroup userGroup2 = new GenericMetricGroup(registry, userGroup1, "world");

	QueryScopeInfo.TaskQueryScopeInfo info1 = (QueryScopeInfo.TaskQueryScopeInfo) userGroup1.createQueryServiceMetricInfo(new DummyCharacterFilter());
	assertEquals("hello", info1.scope);
	assertEquals(jid.toString(), info1.jobID);
	assertEquals(vid.toString(), info1.vertexID);
	assertEquals(4, info1.subtaskIndex);

	QueryScopeInfo.TaskQueryScopeInfo info2 = (QueryScopeInfo.TaskQueryScopeInfo) userGroup2.createQueryServiceMetricInfo(new DummyCharacterFilter());
	assertEquals("hello.world", info2.scope);
	assertEquals(jid.toString(), info2.jobID);
	assertEquals(vid.toString(), info2.vertexID);
	assertEquals(4, info2.subtaskIndex);
}
 
Example 11
Source Project: flink   Source File: OperatorGroupTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateQueryServiceMetricInfo() {
	JobID jid = new JobID();
	JobVertexID vid = new JobVertexID();
	AbstractID eid = new AbstractID();
	OperatorID oid = new OperatorID();
	TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id");
	TaskManagerJobMetricGroup job = new TaskManagerJobMetricGroup(registry, tm, jid, "jobname");
	TaskMetricGroup task = new TaskMetricGroup(registry, job, vid, eid, "taskName", 4, 5);
	OperatorMetricGroup operator = new OperatorMetricGroup(registry, task, oid, "operator");

	QueryScopeInfo.OperatorQueryScopeInfo info = operator.createQueryServiceMetricInfo(new DummyCharacterFilter());
	assertEquals("", info.scope);
	assertEquals(jid.toString(), info.jobID);
	assertEquals(vid.toString(), info.vertexID);
	assertEquals(4, info.subtaskIndex);
	assertEquals("operator", info.operatorName);
}
 
Example 12
Source Project: flink   Source File: BackPressureStatsTrackerImplTest.java    License: Apache License 2.0 6 votes vote down vote up
private ExecutionVertex mockExecutionVertex(
		ExecutionJobVertex jobVertex,
		int subTaskIndex) {

	Execution exec = Mockito.mock(Execution.class);
	Mockito.when(exec.getAttemptId()).thenReturn(new ExecutionAttemptID());

	JobVertexID id = jobVertex.getJobVertexId();

	ExecutionVertex vertex = Mockito.mock(ExecutionVertex.class);
	Mockito.when(vertex.getJobvertexId()).thenReturn(id);
	Mockito.when(vertex.getCurrentExecutionAttempt()).thenReturn(exec);
	Mockito.when(vertex.getParallelSubtaskIndex()).thenReturn(subTaskIndex);

	return vertex;
}
 
Example 13
Source Project: Flink-CEPplus   Source File: TaskLocalStateStoreImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
TaskLocalStateStoreImpl(
	@Nonnull JobID jobID,
	@Nonnull AllocationID allocationID,
	@Nonnull JobVertexID jobVertexID,
	@Nonnegative int subtaskIndex,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull Executor discardExecutor,
	@Nonnull SortedMap<Long, TaskStateSnapshot> storedTaskStateByCheckpointID,
	@Nonnull Object lock) {

	this.jobID = jobID;
	this.allocationID = allocationID;
	this.jobVertexID = jobVertexID;
	this.subtaskIndex = subtaskIndex;
	this.discardExecutor = discardExecutor;
	this.localRecoveryConfig = localRecoveryConfig;
	this.storedTaskStateByCheckpointID = storedTaskStateByCheckpointID;
	this.lock = lock;
	this.disposed = false;
}
 
Example 14
@Test
public void testAbsentBackPressure() throws Exception {
	final Map<String, String> pathParameters = new HashMap<>();
	pathParameters.put(JobIDPathParameter.KEY, TEST_JOB_ID_BACK_PRESSURE_STATS_ABSENT.toString());
	pathParameters.put(JobVertexIdPathParameter.KEY, new JobVertexID().toString());

	final HandlerRequest<EmptyRequestBody, JobVertexMessageParameters> request =
		new HandlerRequest<>(
			EmptyRequestBody.getInstance(),
			new JobVertexMessageParameters(), pathParameters, Collections.emptyMap());

	final CompletableFuture<JobVertexBackPressureInfo> jobVertexBackPressureInfoCompletableFuture =
		jobVertexBackPressureHandler.handleRequest(request, restfulGateway);
	final JobVertexBackPressureInfo jobVertexBackPressureInfo = jobVertexBackPressureInfoCompletableFuture.get();

	assertThat(jobVertexBackPressureInfo.getStatus(), equalTo(VertexBackPressureStatus.DEPRECATED));
}
 
Example 15
Source Project: flink   Source File: FailedCheckpointStatsTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the end to end duration of a failed checkpoint is the duration
 * until the failure.
 */
@Test
public void testEndToEndDuration() throws Exception {
	long duration = 123912931293L;
	long triggerTimestamp = 10123;
	long failureTimestamp = triggerTimestamp + duration;

	Map<JobVertexID, TaskStateStats> taskStats = new HashMap<>();
	JobVertexID jobVertexId = new JobVertexID();
	taskStats.put(jobVertexId, new TaskStateStats(jobVertexId, 1));

	FailedCheckpointStats failed = new FailedCheckpointStats(
		0,
		triggerTimestamp,
		CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION),
		1,
		taskStats,
		0,
		0,
		failureTimestamp,
		null,
		null);

	assertEquals(duration, failed.getEndToEndDuration());
}
 
Example 16
Source Project: Flink-CEPplus   Source File: JobCheckpointingSettings.java    License: Apache License 2.0 5 votes vote down vote up
public JobCheckpointingSettings(
		List<JobVertexID> verticesToTrigger,
		List<JobVertexID> verticesToAcknowledge,
		List<JobVertexID> verticesToConfirm,
		CheckpointCoordinatorConfiguration checkpointCoordinatorConfiguration,
		@Nullable SerializedValue<StateBackend> defaultStateBackend) {

	this(
		verticesToTrigger,
		verticesToAcknowledge,
		verticesToConfirm,
		checkpointCoordinatorConfiguration,
		defaultStateBackend,
		null);
}
 
Example 17
@Test
public void testInterBulkInputLocationPreferencesAreRespected() {
	final ExecutionVertexID producerId = new ExecutionVertexID(new JobVertexID(), 0);
	final ExecutionVertexID consumerId = new ExecutionVertexID(new JobVertexID(), 0);

	final TestingInputsLocationsRetriever inputsLocationsRetriever = new TestingInputsLocationsRetriever.Builder()
		.connectConsumerToProducer(consumerId, producerId)
		.build();

	final ExecutionSlotAllocator executionSlotAllocator = createExecutionSlotAllocator(
		new TestingStateLocationRetriever(),
		inputsLocationsRetriever);

	inputsLocationsRetriever.markScheduled(producerId);
	final List<ExecutionVertexSchedulingRequirements> schedulingRequirementsForProducer =
		createSchedulingRequirements(producerId);
	final Collection<SlotExecutionVertexAssignment> slotExecutionVertexAssignmentsForProducer =
		executionSlotAllocator.allocateSlotsFor(schedulingRequirementsForProducer);
	final SlotExecutionVertexAssignment producerSlotAssignment =
		findSlotAssignmentByExecutionVertexId(producerId, slotExecutionVertexAssignmentsForProducer);

	assertThat(producerSlotAssignment.getLogicalSlotFuture().isDone(), is(true));

	inputsLocationsRetriever.markScheduled(consumerId);
	final List<ExecutionVertexSchedulingRequirements> schedulingRequirementsForConsumer =
		createSchedulingRequirements(consumerId);
	final Collection<SlotExecutionVertexAssignment> slotExecutionVertexAssignmentsForConsumer =
		executionSlotAllocator.allocateSlotsFor(schedulingRequirementsForConsumer);
	final SlotExecutionVertexAssignment consumerSlotAssignment =
		findSlotAssignmentByExecutionVertexId(consumerId, slotExecutionVertexAssignmentsForConsumer);

	assertThat(consumerSlotAssignment.getLogicalSlotFuture().isDone(), is(false));

	inputsLocationsRetriever.assignTaskManagerLocation(producerId);

	assertThat(consumerSlotAssignment.getLogicalSlotFuture().isDone(), is(true));
}
 
Example 18
Source Project: flink   Source File: OperatorGroupTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testIOMetricGroupInstantiation() throws Exception {
	TaskManagerMetricGroup tmGroup = new TaskManagerMetricGroup(registry, "theHostName", "test-tm-id");
	TaskManagerJobMetricGroup jmGroup = new TaskManagerJobMetricGroup(registry, tmGroup, new JobID(), "myJobName");
	TaskMetricGroup taskGroup = new TaskMetricGroup(
		registry, jmGroup, new JobVertexID(), new AbstractID(), "aTaskName", 11, 0);
	OperatorMetricGroup opGroup = new OperatorMetricGroup(registry, taskGroup, new OperatorID(), "myOpName");

	assertNotNull(opGroup.getIOMetricGroup());
	assertNotNull(opGroup.getIOMetricGroup().getNumRecordsInCounter());
	assertNotNull(opGroup.getIOMetricGroup().getNumRecordsOutCounter());
}
 
Example 19
Source Project: flink   Source File: TaskState.java    License: Apache License 2.0 5 votes vote down vote up
public TaskState(JobVertexID jobVertexID, int parallelism, int maxParallelism, int chainLength) {
	Preconditions.checkArgument(
			parallelism <= maxParallelism,
			"Parallelism " + parallelism + " is not smaller or equal to max parallelism " + maxParallelism + ".");
	Preconditions.checkArgument(chainLength > 0, "There has to be at least one operator in the operator chain.");

	this.jobVertexID = jobVertexID;

	this.subtaskStates = new HashMap<>(parallelism);

	this.parallelism = parallelism;
	this.maxParallelism = maxParallelism;
	this.chainLength = chainLength;
}
 
Example 20
Source Project: flink   Source File: ScheduleWithCoLocationHintTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void scheduleWithIntermediateRelease() throws Exception {
	JobVertexID jid1 = new JobVertexID();
	JobVertexID jid2 = new JobVertexID();
	JobVertexID jid3 = new JobVertexID();
	JobVertexID jid4 = new JobVertexID();

	testingSlotProvider.addTaskManager(1);
	testingSlotProvider.addTaskManager(1);

	assertEquals(2, testingSlotProvider.getNumberOfAvailableSlots());

	SlotSharingGroup sharingGroup = new SlotSharingGroup();
	CoLocationConstraint c1 = new CoLocationConstraint(new CoLocationGroup());

	LogicalSlot s1 = testingSlotProvider.allocateSlot(
			new ScheduledUnit(getExecution(jid1, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get();
	LogicalSlot s2 = testingSlotProvider.allocateSlot(
			new ScheduledUnit(getExecution(jid2, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get();

	LogicalSlot sSolo = testingSlotProvider.allocateSlot(new ScheduledUnit(getExecution(jid4, 0, 1, null)), SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get();

	ResourceID taskManager = s1.getTaskManagerLocation().getResourceID();

	s1.releaseSlot();
	s2.releaseSlot();
	sSolo.releaseSlot();

	LogicalSlot sNew = testingSlotProvider.allocateSlot(
			new ScheduledUnit(getExecution(jid3, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get();
	assertEquals(taskManager, sNew.getTaskManagerLocation().getResourceID());

	assertEquals(2, testingSlotProvider.getNumberOfLocalizedAssignments());
	assertEquals(0, testingSlotProvider.getNumberOfNonLocalizedAssignments());
	assertEquals(2, testingSlotProvider.getNumberOfUnconstrainedAssignments());
}
 
Example 21
Source Project: flink   Source File: JobVertexDetailsInfo.java    License: Apache License 2.0 5 votes vote down vote up
@JsonCreator
public JobVertexDetailsInfo(
		@JsonDeserialize(using = JobVertexIDDeserializer.class) @JsonProperty(FIELD_NAME_VERTEX_ID) JobVertexID id,
		@JsonProperty(FIELD_NAME_VERTEX_NAME) String name,
		@JsonProperty(FIELD_NAME_PARALLELISM) int parallelism,
		@JsonProperty(FIELD_NAME_NOW) long now,
		@JsonProperty(FIELD_NAME_SUBTASKS) List<VertexTaskDetail> subtasks) {
	this.id = checkNotNull(id);
	this.name = checkNotNull(name);
	this.parallelism = parallelism;
	this.now = now;
	this.subtasks = checkNotNull(subtasks);
}
 
Example 22
Source Project: flink   Source File: TestingRestfulGateway.java    License: Apache License 2.0 5 votes vote down vote up
public TestingRestfulGateway(
		String address,
		String hostname,
		Function<JobID, CompletableFuture<Acknowledge>> cancelJobFunction,
		Function<JobID, CompletableFuture<ArchivedExecutionGraph>> requestJobFunction,
		Function<JobID, CompletableFuture<JobResult>> requestJobResultFunction,
		Function<JobID, CompletableFuture<JobStatus>> requestJobStatusFunction,
		Supplier<CompletableFuture<MultipleJobsDetails>> requestMultipleJobDetailsSupplier,
		Supplier<CompletableFuture<ClusterOverview>> requestClusterOverviewSupplier,
		Supplier<CompletableFuture<Collection<String>>> requestMetricQueryServiceAddressesSupplier,
		Supplier<CompletableFuture<Collection<Tuple2<ResourceID, String>>>> requestTaskManagerMetricQueryServiceAddressesSupplier,
		BiFunction<JobID, JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOperatorBackPressureStatsFunction,
		BiFunction<JobID, String, CompletableFuture<String>> triggerSavepointFunction,
		BiFunction<JobID, String, CompletableFuture<String>> stopWithSavepointFunction,
		Supplier<CompletableFuture<Acknowledge>> clusterShutdownSupplier,
		TriFunction<JobID, OperatorID, SerializedValue<CoordinationRequest>, CompletableFuture<CoordinationResponse>> deliverCoordinationRequestToCoordinatorFunction) {
	this.address = address;
	this.hostname = hostname;
	this.cancelJobFunction = cancelJobFunction;
	this.requestJobFunction = requestJobFunction;
	this.requestJobResultFunction = requestJobResultFunction;
	this.requestJobStatusFunction = requestJobStatusFunction;
	this.requestMultipleJobDetailsSupplier = requestMultipleJobDetailsSupplier;
	this.requestClusterOverviewSupplier = requestClusterOverviewSupplier;
	this.requestMetricQueryServiceAddressesSupplier = requestMetricQueryServiceAddressesSupplier;
	this.requestTaskManagerMetricQueryServiceAddressesSupplier = requestTaskManagerMetricQueryServiceAddressesSupplier;
	this.requestOperatorBackPressureStatsFunction = requestOperatorBackPressureStatsFunction;
	this.triggerSavepointFunction = triggerSavepointFunction;
	this.stopWithSavepointFunction = stopWithSavepointFunction;
	this.clusterShutdownSupplier = clusterShutdownSupplier;
	this.deliverCoordinationRequestToCoordinatorFunction = deliverCoordinationRequestToCoordinatorFunction;
}
 
Example 23
Source Project: flink   Source File: KvStateServerHandlerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void notifyKvStateUnregistered(JobID jobId,
		JobVertexID jobVertexId,
		KeyGroupRange keyGroupRange,
		String registrationName) {

}
 
Example 24
Source Project: flink   Source File: LegacyJobVertexIdTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testIntroduceLegacyJobVertexIds() throws Exception {
	JobVertexID defaultId = new JobVertexID();
	JobVertexID legacyId1 = new JobVertexID();
	JobVertexID legacyId2 = new JobVertexID();

	JobVertex jobVertex = new JobVertex("test", defaultId, Arrays.asList(legacyId1, legacyId2), new ArrayList<OperatorID>(), new ArrayList<OperatorID>());
	jobVertex.setInvokableClass(AbstractInvokable.class);

	ExecutionGraph executionGraph = new ExecutionGraph(
		mock(ScheduledExecutorService.class),
		mock(Executor.class),
		new JobID(),
		"test",
		mock(Configuration.class),
		mock(SerializedValue.class),
		Time.seconds(1),
		mock(RestartStrategy.class),
		mock(SlotProvider.class));

	ExecutionJobVertex executionJobVertex =
			new ExecutionJobVertex(executionGraph, jobVertex, 1, Time.seconds(1));

	Map<JobVertexID, ExecutionJobVertex> idToVertex = new HashMap<>();
	idToVertex.put(executionJobVertex.getJobVertexId(), executionJobVertex);

	Assert.assertEquals(executionJobVertex, idToVertex.get(defaultId));
	Assert.assertNull(idToVertex.get(legacyId1));
	Assert.assertNull(idToVertex.get(legacyId2));

	idToVertex = ExecutionJobVertex.includeLegacyJobVertexIDs(idToVertex);

	Assert.assertEquals(3, idToVertex.size());
	Assert.assertEquals(executionJobVertex, idToVertex.get(defaultId));
	Assert.assertEquals(executionJobVertex, idToVertex.get(legacyId1));
	Assert.assertEquals(executionJobVertex, idToVertex.get(legacyId2));
}
 
Example 25
Source Project: Flink-CEPplus   Source File: UnregisteredMetricGroups.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public TaskMetricGroup addTaskForJob(
	final JobID jobId,
	final String jobName,
	final JobVertexID jobVertexId,
	final ExecutionAttemptID executionAttemptId,
	final String taskName,
	final int subtaskIndex,
	final int attemptNumber) {
	return createUnregisteredTaskMetricGroup();
}
 
Example 26
Source Project: flink   Source File: IntermediateResultPartitionTest.java    License: Apache License 2.0 5 votes vote down vote up
private static IntermediateResult createResult(
		ResultPartitionType resultPartitionType,
		int producerCount) throws Exception {

	ExecutionJobVertex jobVertex = getExecutionJobVertex(new JobVertexID(), new DirectScheduledExecutorService());
	IntermediateResult result =
			new IntermediateResult(new IntermediateDataSetID(), jobVertex, producerCount, resultPartitionType);
	for (int i = 0; i < producerCount; i++) {
		// Generate result partition in the result
		new ExecutionVertex(jobVertex, i, new IntermediateResult[]{result}, Time.minutes(1));
	}

	return result;
}
 
Example 27
Source Project: flink   Source File: ScheduledUnit.java    License: Apache License 2.0 5 votes vote down vote up
public ScheduledUnit(
	@Nullable Execution task,
	JobVertexID jobVertexId,
	@Nullable SlotSharingGroupId slotSharingGroupId,
	@Nullable CoLocationConstraint coLocationConstraint) {

	this.vertexExecution = task;
	this.jobVertexId = Preconditions.checkNotNull(jobVertexId);
	this.slotSharingGroupId = slotSharingGroupId;
	this.coLocationConstraint = coLocationConstraint;

}
 
Example 28
Source Project: flink   Source File: CheckpointStatsTracker.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new pending checkpoint tracker.
 *
 * @param checkpointId ID of the checkpoint.
 * @param triggerTimestamp Trigger timestamp of the checkpoint.
 * @param props The checkpoint properties.
 * @return Tracker for statistics gathering.
 */
PendingCheckpointStats reportPendingCheckpoint(
		long checkpointId,
		long triggerTimestamp,
		CheckpointProperties props) {

	ConcurrentHashMap<JobVertexID, TaskStateStats> taskStateStats = createEmptyTaskStateStatsMap();

	PendingCheckpointStats pending = new PendingCheckpointStats(
			checkpointId,
			triggerTimestamp,
			props,
			totalSubtaskCount,
			taskStateStats,
			new PendingCheckpointStatsCallback());

	statsReadWriteLock.lock();
	try {
		counts.incrementInProgressCheckpoints();
		history.addInProgressCheckpoint(pending);

		dirty = true;
	} finally {
		statsReadWriteLock.unlock();
	}

	return pending;
}
 
Example 29
/**
 * Verifies that each {@link JobVertexID} of the {@link JobGraph} is contained in the given map
 * and mapped to the same vertex name.
 */
private void verifyIdsEqual(JobGraph jobGraph, Map<JobVertexID, String> ids) {
	// Verify same number of vertices
	assertEquals(jobGraph.getNumberOfVertices(), ids.size());

	// Verify that all IDs->name mappings are identical
	for (JobVertex vertex : jobGraph.getVertices()) {
		String expectedName = ids.get(vertex.getID());
		assertNotNull(expectedName);
		assertEquals(expectedName, vertex.getName());
	}
}
 
Example 30
Source Project: flink   Source File: ExecutionVertexCancelTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCancelCallFails() {
	try {
		final JobVertexID jid = new JobVertexID();
		final ExecutionJobVertex ejv = ExecutionGraphTestUtils.getExecutionJobVertex(jid, new DirectScheduledExecutorService());

		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
				AkkaUtils.getDefaultTimeout());

		LogicalSlot slot = new TestingLogicalSlotBuilder().setTaskManagerGateway(new CancelSequenceSimpleAckingTaskManagerGateway(0)).createTestingLogicalSlot();

		setVertexResource(vertex, slot);
		setVertexState(vertex, ExecutionState.RUNNING);

		assertEquals(ExecutionState.RUNNING, vertex.getExecutionState());

		vertex.cancel();

		// Callback fails, leading to CANCELED
		assertEquals(ExecutionState.CANCELED, vertex.getExecutionState());

		assertFalse(slot.isAlive());

		assertTrue(vertex.getStateTimestamp(ExecutionState.CREATED) > 0);
		assertTrue(vertex.getStateTimestamp(ExecutionState.CANCELING) > 0);
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}