org.apache.flink.runtime.jobgraph.JobVertexID Java Examples
The following examples show how to use
org.apache.flink.runtime.jobgraph.JobVertexID.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TaskLocalStateStoreImpl.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@VisibleForTesting TaskLocalStateStoreImpl( @Nonnull JobID jobID, @Nonnull AllocationID allocationID, @Nonnull JobVertexID jobVertexID, @Nonnegative int subtaskIndex, @Nonnull LocalRecoveryConfig localRecoveryConfig, @Nonnull Executor discardExecutor, @Nonnull SortedMap<Long, TaskStateSnapshot> storedTaskStateByCheckpointID, @Nonnull Object lock) { this.jobID = jobID; this.allocationID = allocationID; this.jobVertexID = jobVertexID; this.subtaskIndex = subtaskIndex; this.discardExecutor = discardExecutor; this.localRecoveryConfig = localRecoveryConfig; this.storedTaskStateByCheckpointID = storedTaskStateByCheckpointID; this.lock = lock; this.disposed = false; }
Example #2
Source File: CheckpointCoordinatorTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
static TaskStateSnapshot mockSubtaskState( JobVertexID jobVertexID, int index, KeyGroupRange keyGroupRange) throws IOException { OperatorStateHandle partitionableState = generatePartitionableStateHandle(jobVertexID, index, 2, 8, false); KeyGroupsStateHandle partitionedKeyGroupState = generateKeyGroupState(jobVertexID, keyGroupRange, false); TaskStateSnapshot subtaskStates = spy(new TaskStateSnapshot()); OperatorSubtaskState subtaskState = spy(new OperatorSubtaskState( partitionableState, null, partitionedKeyGroupState, null) ); subtaskStates.putSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID), subtaskState); return subtaskStates; }
Example #3
Source File: CheckpointCoordinatorTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public static OperatorStateHandle generatePartitionableStateHandle( JobVertexID jobVertexID, int index, int namedStates, int partitionsPerState, boolean rawState) throws IOException { Map<String, List<? extends Serializable>> statesListsMap = new HashMap<>(namedStates); for (int i = 0; i < namedStates; ++i) { List<Integer> testStatesLists = new ArrayList<>(partitionsPerState); // generate state int seed = jobVertexID.hashCode() * index + i * namedStates; if (rawState) { seed = (seed + 1) * 31; } Random random = new Random(seed); for (int j = 0; j < partitionsPerState; ++j) { int simulatedStateValue = random.nextInt(); testStatesLists.add(simulatedStateValue); } statesListsMap.put("state-" + i, testStatesLists); } return generatePartitionableStateHandle(statesListsMap); }
Example #4
Source File: JobVertexBackPressureHandlerTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testAbsentBackPressure() throws Exception { final Map<String, String> pathParameters = new HashMap<>(); pathParameters.put(JobIDPathParameter.KEY, TEST_JOB_ID_BACK_PRESSURE_STATS_ABSENT.toString()); pathParameters.put(JobVertexIdPathParameter.KEY, new JobVertexID().toString()); final HandlerRequest<EmptyRequestBody, JobVertexMessageParameters> request = new HandlerRequest<>( EmptyRequestBody.getInstance(), new JobVertexMessageParameters(), pathParameters, Collections.emptyMap()); final CompletableFuture<JobVertexBackPressureInfo> jobVertexBackPressureInfoCompletableFuture = jobVertexBackPressureHandler.handleRequest(request, restfulGateway); final JobVertexBackPressureInfo jobVertexBackPressureInfo = jobVertexBackPressureInfoCompletableFuture.get(); assertThat(jobVertexBackPressureInfo.getStatus(), equalTo(VertexBackPressureStatus.DEPRECATED)); }
Example #5
Source File: MetricGroupTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testCreateQueryServiceMetricInfo() { JobID jid = new JobID(); JobVertexID vid = new JobVertexID(); AbstractID eid = new AbstractID(); MetricRegistryImpl registry = new MetricRegistryImpl(defaultMetricRegistryConfiguration); TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id"); TaskManagerJobMetricGroup job = new TaskManagerJobMetricGroup(registry, tm, jid, "jobname"); TaskMetricGroup task = new TaskMetricGroup(registry, job, vid, eid, "taskName", 4, 5); GenericMetricGroup userGroup1 = new GenericMetricGroup(registry, task, "hello"); GenericMetricGroup userGroup2 = new GenericMetricGroup(registry, userGroup1, "world"); QueryScopeInfo.TaskQueryScopeInfo info1 = (QueryScopeInfo.TaskQueryScopeInfo) userGroup1.createQueryServiceMetricInfo(new DummyCharacterFilter()); assertEquals("hello", info1.scope); assertEquals(jid.toString(), info1.jobID); assertEquals(vid.toString(), info1.vertexID); assertEquals(4, info1.subtaskIndex); QueryScopeInfo.TaskQueryScopeInfo info2 = (QueryScopeInfo.TaskQueryScopeInfo) userGroup2.createQueryServiceMetricInfo(new DummyCharacterFilter()); assertEquals("hello.world", info2.scope); assertEquals(jid.toString(), info2.jobID); assertEquals(vid.toString(), info2.vertexID); assertEquals(4, info2.subtaskIndex); }
Example #6
Source File: OperatorGroupTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testCreateQueryServiceMetricInfo() { JobID jid = new JobID(); JobVertexID vid = new JobVertexID(); AbstractID eid = new AbstractID(); OperatorID oid = new OperatorID(); TaskManagerMetricGroup tm = new TaskManagerMetricGroup(registry, "host", "id"); TaskManagerJobMetricGroup job = new TaskManagerJobMetricGroup(registry, tm, jid, "jobname"); TaskMetricGroup task = new TaskMetricGroup(registry, job, vid, eid, "taskName", 4, 5); OperatorMetricGroup operator = new OperatorMetricGroup(registry, task, oid, "operator"); QueryScopeInfo.OperatorQueryScopeInfo info = operator.createQueryServiceMetricInfo(new DummyCharacterFilter()); assertEquals("", info.scope); assertEquals(jid.toString(), info.jobID); assertEquals(vid.toString(), info.vertexID); assertEquals(4, info.subtaskIndex); assertEquals("operator", info.operatorName); }
Example #7
Source File: CheckpointCoordinatorTest.java From flink with Apache License 2.0 | 6 votes |
public static ChainedStateHandle<OperatorStateHandle> generateChainedPartitionableStateHandle( JobVertexID jobVertexID, int index, int namedStates, int partitionsPerState, boolean rawState) throws IOException { Map<String, List<? extends Serializable>> statesListsMap = new HashMap<>(namedStates); for (int i = 0; i < namedStates; ++i) { List<Integer> testStatesLists = new ArrayList<>(partitionsPerState); // generate state int seed = jobVertexID.hashCode() * index + i * namedStates; if (rawState) { seed = (seed + 1) * 31; } Random random = new Random(seed); for (int j = 0; j < partitionsPerState; ++j) { int simulatedStateValue = random.nextInt(); testStatesLists.add(simulatedStateValue); } statesListsMap.put("state-" + i, testStatesLists); } return ChainedStateHandle.wrapSingleHandle(generatePartitionableStateHandle(statesListsMap)); }
Example #8
Source File: FailedCheckpointStatsTest.java From flink with Apache License 2.0 | 6 votes |
/** * Tests that the end to end duration of a failed checkpoint is the duration * until the failure. */ @Test public void testEndToEndDuration() throws Exception { long duration = 123912931293L; long triggerTimestamp = 10123; long failureTimestamp = triggerTimestamp + duration; Map<JobVertexID, TaskStateStats> taskStats = new HashMap<>(); JobVertexID jobVertexId = new JobVertexID(); taskStats.put(jobVertexId, new TaskStateStats(jobVertexId, 1)); FailedCheckpointStats failed = new FailedCheckpointStats( 0, triggerTimestamp, CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION), 1, taskStats, 0, 0, failureTimestamp, null, null); assertEquals(duration, failed.getEndToEndDuration()); }
Example #9
Source File: CheckpointMetadataLoadingTest.java From flink with Apache License 2.0 | 6 votes |
/** * Tests correct savepoint loading. */ @Test public void testAllStateRestored() throws Exception { final JobID jobId = new JobID(); final OperatorID operatorId = new OperatorID(); final long checkpointId = Integer.MAX_VALUE + 123123L; final int parallelism = 128128; final CompletedCheckpointStorageLocation testSavepoint = createSavepointWithOperatorSubtaskState(checkpointId, operatorId, parallelism); final Map<JobVertexID, ExecutionJobVertex> tasks = createTasks(operatorId, parallelism, parallelism); final CompletedCheckpoint loaded = Checkpoints.loadAndValidateCheckpoint(jobId, tasks, testSavepoint, cl, false); assertEquals(jobId, loaded.getJobId()); assertEquals(checkpointId, loaded.getCheckpointID()); }
Example #10
Source File: AbstractExecutionSlotAllocatorTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testComputeAllPriorAllocationIds() { final List<AllocationID> expectAllocationIds = Arrays.asList(new AllocationID(), new AllocationID()); final List<ExecutionVertexSchedulingRequirements> testSchedulingRequirements = Arrays.asList( new ExecutionVertexSchedulingRequirements.Builder(). withExecutionVertexId(new ExecutionVertexID(new JobVertexID(), 0)). withPreviousAllocationId(expectAllocationIds.get(0)). build(), new ExecutionVertexSchedulingRequirements.Builder(). withExecutionVertexId(new ExecutionVertexID(new JobVertexID(), 1)). withPreviousAllocationId(expectAllocationIds.get(0)). build(), new ExecutionVertexSchedulingRequirements.Builder(). withExecutionVertexId(new ExecutionVertexID(new JobVertexID(), 2)). withPreviousAllocationId(expectAllocationIds.get(1)). build(), new ExecutionVertexSchedulingRequirements.Builder(). withExecutionVertexId(new ExecutionVertexID(new JobVertexID(), 3)). build() ); final Set<AllocationID> allPriorAllocationIds = AbstractExecutionSlotAllocator.computeAllPriorAllocationIds(testSchedulingRequirements); assertThat(allPriorAllocationIds, containsInAnyOrder(expectAllocationIds.toArray())); }
Example #11
Source File: BackPressureStatsTrackerImplTest.java From flink with Apache License 2.0 | 6 votes |
private ExecutionVertex mockExecutionVertex( ExecutionJobVertex jobVertex, int subTaskIndex) { Execution exec = Mockito.mock(Execution.class); Mockito.when(exec.getAttemptId()).thenReturn(new ExecutionAttemptID()); JobVertexID id = jobVertex.getJobVertexId(); ExecutionVertex vertex = Mockito.mock(ExecutionVertex.class); Mockito.when(vertex.getJobvertexId()).thenReturn(id); Mockito.when(vertex.getCurrentExecutionAttempt()).thenReturn(exec); Mockito.when(vertex.getParallelSubtaskIndex()).thenReturn(subTaskIndex); return vertex; }
Example #12
Source File: CheckpointStatistics.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@JsonCreator public PendingCheckpointStatistics( @JsonProperty(FIELD_NAME_ID) long id, @JsonProperty(FIELD_NAME_STATUS) CheckpointStatsStatus status, @JsonProperty(FIELD_NAME_IS_SAVEPOINT) boolean savepoint, @JsonProperty(FIELD_NAME_TRIGGER_TIMESTAMP) long triggerTimestamp, @JsonProperty(FIELD_NAME_LATEST_ACK_TIMESTAMP) long latestAckTimestamp, @JsonProperty(FIELD_NAME_STATE_SIZE) long stateSize, @JsonProperty(FIELD_NAME_DURATION) long duration, @JsonProperty(FIELD_NAME_ALIGNMENT_BUFFERED) long alignmentBuffered, @JsonProperty(FIELD_NAME_NUM_SUBTASKS) int numSubtasks, @JsonProperty(FIELD_NAME_NUM_ACK_SUBTASKS) int numAckSubtasks, @JsonDeserialize(keyUsing = JobVertexIDKeyDeserializer.class) @JsonProperty(FIELD_NAME_TASKS) Map<JobVertexID, TaskCheckpointStatistics> checkpointingStatisticsPerTask) { super( id, status, savepoint, triggerTimestamp, latestAckTimestamp, stateSize, duration, alignmentBuffered, numSubtasks, numAckSubtasks, checkpointingStatisticsPerTask); }
Example #13
Source File: SchedulerTestUtils.java From flink with Apache License 2.0 | 6 votes |
public static Execution getTestVertex(Collection<CompletableFuture<TaskManagerLocation>> preferredLocationFutures) { ExecutionJobVertex executionJobVertex = mock(ExecutionJobVertex.class); ExecutionVertex vertex = mock(ExecutionVertex.class); when(vertex.getPreferredLocationsBasedOnInputs()).thenReturn(preferredLocationFutures); when(vertex.getPreferredLocations()).thenReturn(preferredLocationFutures); when(vertex.getJobId()).thenReturn(new JobID()); when(vertex.toString()).thenReturn("TEST-VERTEX"); when(vertex.getJobVertex()).thenReturn(executionJobVertex); when(vertex.getJobvertexId()).thenReturn(new JobVertexID()); Execution execution = mock(Execution.class); when(execution.getVertex()).thenReturn(vertex); when(execution.calculatePreferredLocations(any(LocationPreferenceConstraint.class))).thenCallRealMethod(); return execution; }
Example #14
Source File: OperatorGroupTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testGenerateScopeDefault() throws Exception { TaskManagerMetricGroup tmGroup = new TaskManagerMetricGroup(registry, "theHostName", "test-tm-id"); TaskManagerJobMetricGroup jmGroup = new TaskManagerJobMetricGroup(registry, tmGroup, new JobID(), "myJobName"); TaskMetricGroup taskGroup = new TaskMetricGroup( registry, jmGroup, new JobVertexID(), new AbstractID(), "aTaskName", 11, 0); OperatorMetricGroup opGroup = new OperatorMetricGroup(registry, taskGroup, new OperatorID(), "myOpName"); assertArrayEquals( new String[] { "theHostName", "taskmanager", "test-tm-id", "myJobName", "myOpName", "11" }, opGroup.getScopeComponents()); assertEquals( "theHostName.taskmanager.test-tm-id.myJobName.myOpName.11.name", opGroup.getMetricIdentifier("name")); }
Example #15
Source File: SchedulerTestBase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public int getNumberOfAvailableSlotsForGroup(SlotSharingGroupId slotSharingGroupId, JobVertexID jobVertexId) { final SlotSharingManager multiTaskSlotManager = slotSharingManagersMap.get(slotSharingGroupId); if (multiTaskSlotManager != null) { int availableSlots = 0; for (SlotSharingManager.MultiTaskSlot multiTaskSlot : multiTaskSlotManager.getResolvedRootSlots()) { if (!multiTaskSlot.contains(jobVertexId)) { availableSlots++; } } return availableSlots; } else { throw new FlinkRuntimeException("No MultiTaskSlotmanager registered under " + slotSharingGroupId + '.'); } }
Example #16
Source File: ExecutionTest.java From flink with Apache License 2.0 | 5 votes |
@Nonnull private JobVertex createNoOpJobVertex() { final JobVertex jobVertex = new JobVertex("Test vertex", new JobVertexID()); jobVertex.setInvokableClass(NoOpInvokable.class); return jobVertex; }
Example #17
Source File: TestingRestfulGateway.java From flink with Apache License 2.0 | 5 votes |
public TestingRestfulGateway( String address, String hostname, Function<JobID, CompletableFuture<Acknowledge>> cancelJobFunction, Function<JobID, CompletableFuture<ArchivedExecutionGraph>> requestJobFunction, Function<JobID, CompletableFuture<JobResult>> requestJobResultFunction, Function<JobID, CompletableFuture<JobStatus>> requestJobStatusFunction, Supplier<CompletableFuture<MultipleJobsDetails>> requestMultipleJobDetailsSupplier, Supplier<CompletableFuture<ClusterOverview>> requestClusterOverviewSupplier, Supplier<CompletableFuture<Collection<String>>> requestMetricQueryServiceAddressesSupplier, Supplier<CompletableFuture<Collection<Tuple2<ResourceID, String>>>> requestTaskManagerMetricQueryServiceAddressesSupplier, BiFunction<JobID, JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOperatorBackPressureStatsFunction, BiFunction<JobID, String, CompletableFuture<String>> triggerSavepointFunction, BiFunction<JobID, String, CompletableFuture<String>> stopWithSavepointFunction) { this.address = address; this.hostname = hostname; this.cancelJobFunction = cancelJobFunction; this.requestJobFunction = requestJobFunction; this.requestJobResultFunction = requestJobResultFunction; this.requestJobStatusFunction = requestJobStatusFunction; this.requestMultipleJobDetailsSupplier = requestMultipleJobDetailsSupplier; this.requestClusterOverviewSupplier = requestClusterOverviewSupplier; this.requestMetricQueryServiceAddressesSupplier = requestMetricQueryServiceAddressesSupplier; this.requestTaskManagerMetricQueryServiceAddressesSupplier = requestTaskManagerMetricQueryServiceAddressesSupplier; this.requestOperatorBackPressureStatsFunction = requestOperatorBackPressureStatsFunction; this.triggerSavepointFunction = triggerSavepointFunction; this.stopWithSavepointFunction = stopWithSavepointFunction; }
Example #18
Source File: PipelinedRegionExecutionViewTest.java From flink with Apache License 2.0 | 5 votes |
@Test(expected = IllegalArgumentException.class) public void finishingUnknownVertexThrowsException() { final PipelinedRegion from = PipelinedRegion.from(TEST_EXECUTION_VERTEX_ID); final PipelinedRegionExecutionView pipelinedRegionExecutionView = new PipelinedRegionExecutionView(from); final ExecutionVertexID unknownVertexId = new ExecutionVertexID(new JobVertexID(), 0); pipelinedRegionExecutionView.vertexFinished(unknownVertexId); }
Example #19
Source File: DummyScheduledUnit.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public DummyScheduledUnit() { super( null, new JobVertexID(), null, null); }
Example #20
Source File: TaskInformation.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public TaskInformation( JobVertexID jobVertexId, String taskName, int numberOfSubtasks, int maxNumberOfSubtaks, String invokableClassName, Configuration taskConfiguration) { this.jobVertexId = Preconditions.checkNotNull(jobVertexId); this.taskName = Preconditions.checkNotNull(taskName); this.numberOfSubtasks = Preconditions.checkNotNull(numberOfSubtasks); this.maxNumberOfSubtaks = Preconditions.checkNotNull(maxNumberOfSubtaks); this.invokableClassName = Preconditions.checkNotNull(invokableClassName); this.taskConfiguration = Preconditions.checkNotNull(taskConfiguration); }
Example #21
Source File: KvStateLocationRegistryTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private ExecutionJobVertex createJobVertex(int maxParallelism) { JobVertexID id = new JobVertexID(); ExecutionJobVertex vertex = mock(ExecutionJobVertex.class); when(vertex.getJobVertexId()).thenReturn(id); when(vertex.getMaxParallelism()).thenReturn(maxParallelism); return vertex; }
Example #22
Source File: LegacyJobVertexIdTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testIntroduceLegacyJobVertexIds() throws Exception { JobVertexID defaultId = new JobVertexID(); JobVertexID legacyId1 = new JobVertexID(); JobVertexID legacyId2 = new JobVertexID(); JobVertex jobVertex = new JobVertex("test", defaultId, Arrays.asList(legacyId1, legacyId2), new ArrayList<OperatorID>(), new ArrayList<OperatorID>()); jobVertex.setInvokableClass(AbstractInvokable.class); ExecutionGraph executionGraph = new ExecutionGraph( mock(ScheduledExecutorService.class), mock(Executor.class), new JobID(), "test", mock(Configuration.class), mock(SerializedValue.class), Time.seconds(1), mock(RestartStrategy.class), mock(SlotProvider.class)); ExecutionJobVertex executionJobVertex = new ExecutionJobVertex(executionGraph, jobVertex, 1, Time.seconds(1)); Map<JobVertexID, ExecutionJobVertex> idToVertex = new HashMap<>(); idToVertex.put(executionJobVertex.getJobVertexId(), executionJobVertex); Assert.assertEquals(executionJobVertex, idToVertex.get(defaultId)); Assert.assertNull(idToVertex.get(legacyId1)); Assert.assertNull(idToVertex.get(legacyId2)); idToVertex = ExecutionJobVertex.includeLegacyJobVertexIDs(idToVertex); Assert.assertEquals(3, idToVertex.size()); Assert.assertEquals(executionJobVertex, idToVertex.get(defaultId)); Assert.assertEquals(executionJobVertex, idToVertex.get(legacyId1)); Assert.assertEquals(executionJobVertex, idToVertex.get(legacyId2)); }
Example #23
Source File: CheckpointStatsTrackerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Tests access to the snapshotting settings. */ @Test public void testGetSnapshottingSettings() throws Exception { ExecutionJobVertex jobVertex = mock(ExecutionJobVertex.class); when(jobVertex.getJobVertexId()).thenReturn(new JobVertexID()); when(jobVertex.getParallelism()).thenReturn(1); JobCheckpointingSettings snapshottingSettings = new JobCheckpointingSettings( Collections.singletonList(new JobVertexID()), Collections.singletonList(new JobVertexID()), Collections.singletonList(new JobVertexID()), new CheckpointCoordinatorConfiguration( 181238123L, 19191992L, 191929L, 123, CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION, false ), null); CheckpointStatsTracker tracker = new CheckpointStatsTracker( 0, Collections.singletonList(jobVertex), snapshottingSettings.getCheckpointCoordinatorConfiguration(), new UnregisteredMetricsGroup()); assertEquals(snapshottingSettings.getCheckpointCoordinatorConfiguration(), tracker.getJobCheckpointingConfiguration()); }
Example #24
Source File: StateAssignmentOperation.java From flink with Apache License 2.0 | 5 votes |
public StateAssignmentOperation( long restoreCheckpointId, Map<JobVertexID, ExecutionJobVertex> tasks, Map<OperatorID, OperatorState> operatorStates, boolean allowNonRestoredState) { this.restoreCheckpointId = restoreCheckpointId; this.tasks = Preconditions.checkNotNull(tasks); this.operatorStates = Preconditions.checkNotNull(operatorStates); this.allowNonRestoredState = allowNonRestoredState; }
Example #25
Source File: StreamingJobGraphGeneratorNodeHashTest.java From flink with Apache License 2.0 | 5 votes |
/** * Verifies that each {@link JobVertexID} of the {@link JobGraph} is contained in the given map * and mapped to the same vertex name. */ private void verifyIdsEqual(JobGraph jobGraph, Map<JobVertexID, String> ids) { // Verify same number of vertices assertEquals(jobGraph.getNumberOfVertices(), ids.size()); // Verify that all IDs->name mappings are identical for (JobVertex vertex : jobGraph.getVertices()) { String expectedName = ids.get(vertex.getID()); assertNotNull(expectedName); assertEquals(expectedName, vertex.getName()); } }
Example #26
Source File: CheckpointStatsTracker.java From flink with Apache License 2.0 | 5 votes |
/** * Creates a new pending checkpoint tracker. * * @param checkpointId ID of the checkpoint. * @param triggerTimestamp Trigger timestamp of the checkpoint. * @param props The checkpoint properties. * @return Tracker for statistics gathering. */ PendingCheckpointStats reportPendingCheckpoint( long checkpointId, long triggerTimestamp, CheckpointProperties props) { ConcurrentHashMap<JobVertexID, TaskStateStats> taskStateStats = createEmptyTaskStateStatsMap(); PendingCheckpointStats pending = new PendingCheckpointStats( checkpointId, triggerTimestamp, props, totalSubtaskCount, taskStateStats, new PendingCheckpointStatsCallback()); statsReadWriteLock.lock(); try { counts.incrementInProgressCheckpoints(); history.addInProgressCheckpoint(pending); dirty = true; } finally { statsReadWriteLock.unlock(); } return pending; }
Example #27
Source File: ArchivedJobGenerationUtils.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private static void generateArchivedJob() throws Exception { // Attempt StringifiedAccumulatorResult acc1 = new StringifiedAccumulatorResult("name1", "type1", "value1"); StringifiedAccumulatorResult acc2 = new StringifiedAccumulatorResult("name2", "type2", "value2"); TaskManagerLocation location = new TaskManagerLocation(new ResourceID("hello"), InetAddress.getLocalHost(), 1234); AllocationID allocationID = new AllocationID(42L, 43L); originalAttempt = new ArchivedExecutionBuilder() .setStateTimestamps(new long[]{1, 2, 3, 4, 5, 6, 7, 8, 9}) .setParallelSubtaskIndex(1) .setAttemptNumber(0) .setAssignedResourceLocation(location) .setAssignedAllocationID(allocationID) .setUserAccumulators(new StringifiedAccumulatorResult[]{acc1, acc2}) .setState(ExecutionState.FINISHED) .setFailureCause("attemptException") .build(); // Subtask originalSubtask = new ArchivedExecutionVertexBuilder() .setSubtaskIndex(originalAttempt.getParallelSubtaskIndex()) .setTaskNameWithSubtask("hello(1/1)") .setCurrentExecution(originalAttempt) .build(); // Task originalTask = new ArchivedExecutionJobVertexBuilder() .setTaskVertices(new ArchivedExecutionVertex[]{originalSubtask}) .build(); // Job Map<JobVertexID, ArchivedExecutionJobVertex> tasks = new HashMap<>(); tasks.put(originalTask.getJobVertexId(), originalTask); originalJob = new ArchivedExecutionGraphBuilder() .setJobID(new JobID()) .setTasks(tasks) .setFailureCause(new ErrorInfo(new Exception("jobException"), originalAttempt.getStateTimestamp(ExecutionState.FAILED))) .setState(JobStatus.FINISHED) .setStateTimestamps(new long[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}) .setArchivedUserAccumulators(new StringifiedAccumulatorResult[]{acc1, acc2}) .build(); }
Example #28
Source File: ScheduledUnit.java From flink with Apache License 2.0 | 5 votes |
public ScheduledUnit( @Nullable Execution task, JobVertexID jobVertexId, @Nullable SlotSharingGroupId slotSharingGroupId, @Nullable CoLocationConstraint coLocationConstraint) { this.vertexExecution = task; this.jobVertexId = Preconditions.checkNotNull(jobVertexId); this.slotSharingGroupId = slotSharingGroupId; this.coLocationConstraint = coLocationConstraint; }
Example #29
Source File: DefaultExecutionSlotAllocatorTest.java From flink with Apache License 2.0 | 5 votes |
/** * Tests that cancels an execution vertex which is not existed. */ @Test public void testCancelNonExistingExecutionVertex() { final DefaultExecutionSlotAllocator executionSlotAllocator = createExecutionSlotAllocator(); ExecutionVertexID inValidExecutionVertexId = new ExecutionVertexID(new JobVertexID(), 0); executionSlotAllocator.cancel(inValidExecutionVertexId); assertThat(slotProvider.getCancelledSlotRequestIds(), is(empty())); }
Example #30
Source File: IntermediateResultPartitionTest.java From flink with Apache License 2.0 | 5 votes |
private static IntermediateResult createResult( ResultPartitionType resultPartitionType, int producerCount) throws Exception { ExecutionJobVertex jobVertex = getExecutionJobVertex(new JobVertexID(), new DirectScheduledExecutorService()); IntermediateResult result = new IntermediateResult(new IntermediateDataSetID(), jobVertex, producerCount, resultPartitionType); for (int i = 0; i < producerCount; i++) { // Generate result partition in the result new ExecutionVertex(jobVertex, i, new IntermediateResult[]{result}, Time.minutes(1)); } return result; }