Java Code Examples for org.apache.flink.runtime.state.SharedStateRegistry

The following examples show how to use org.apache.flink.runtime.state.SharedStateRegistry. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: CompletedCheckpointStoreTest.java    License: Apache License 2.0 6 votes vote down vote up
public static TestCompletedCheckpoint createCheckpoint(
	int id,
	SharedStateRegistry sharedStateRegistry) throws IOException {

	int numberOfStates = 4;
	CheckpointProperties props = CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION);

	OperatorID operatorID = new OperatorID();

	Map<OperatorID, OperatorState> operatorGroupState = new HashMap<>();
	OperatorState operatorState = new OperatorState(operatorID, numberOfStates, numberOfStates);
	operatorGroupState.put(operatorID, operatorState);

	for (int i = 0; i < numberOfStates; i++) {
		OperatorSubtaskState subtaskState =
			new TestOperatorSubtaskState();

		operatorState.putState(i, subtaskState);
	}

	operatorState.registerSharedStates(sharedStateRegistry);

	return new TestCompletedCheckpoint(new JobID(), id, 0, operatorGroupState, props);
}
 
Example 2
/**
 * Tests adding and getting a checkpoint.
 */
@Test
public void testAddAndGetLatestCheckpoint() throws Exception {
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	CompletedCheckpointStore checkpoints = createCompletedCheckpoints(4);
	
	// Empty state
	assertEquals(0, checkpoints.getNumberOfRetainedCheckpoints());
	assertEquals(0, checkpoints.getAllCheckpoints().size());

	TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] {
			createCheckpoint(0, sharedStateRegistry), createCheckpoint(1, sharedStateRegistry) };

	// Add and get latest
	checkpoints.addCheckpoint(expected[0]);
	assertEquals(1, checkpoints.getNumberOfRetainedCheckpoints());
	verifyCheckpoint(expected[0], checkpoints.getLatestCheckpoint());

	checkpoints.addCheckpoint(expected[1]);
	assertEquals(2, checkpoints.getNumberOfRetainedCheckpoints());
	verifyCheckpoint(expected[1], checkpoints.getLatestCheckpoint());
}
 
Example 3
/**
 * Tests that all added checkpoints are returned.
 */
@Test
public void testGetAllCheckpoints() throws Exception {
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	CompletedCheckpointStore checkpoints = createCompletedCheckpoints(4);

	TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] {
			createCheckpoint(0, sharedStateRegistry), createCheckpoint(1, sharedStateRegistry),
			createCheckpoint(2, sharedStateRegistry), createCheckpoint(3, sharedStateRegistry)
	};

	for (TestCompletedCheckpoint checkpoint : expected) {
		checkpoints.addCheckpoint(checkpoint);
	}

	List<CompletedCheckpoint> actual = checkpoints.getAllCheckpoints();

	assertEquals(expected.length, actual.size());

	for (int i = 0; i < expected.length; i++) {
		assertEquals(expected[i], actual.get(i));
	}
}
 
Example 4
public static TestCompletedCheckpoint createCheckpoint(
	int id,
	SharedStateRegistry sharedStateRegistry) throws IOException {

	int numberOfStates = 4;
	CheckpointProperties props = CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION);

	OperatorID operatorID = new OperatorID();

	Map<OperatorID, OperatorState> operatorGroupState = new HashMap<>();
	OperatorState operatorState = new OperatorState(operatorID, numberOfStates, numberOfStates);
	operatorGroupState.put(operatorID, operatorState);

	for (int i = 0; i < numberOfStates; i++) {
		OperatorSubtaskState subtaskState =
			new TestOperatorSubtaskState();

		operatorState.putState(i, subtaskState);
	}

	operatorState.registerSharedStates(sharedStateRegistry);

	return new TestCompletedCheckpoint(new JobID(), id, 0, operatorGroupState, props);
}
 
Example 5
/**
 * Tests that checkpoints are discarded when the completed checkpoint store is shut
 * down with a globally terminal state.
 */
@Test
public void testDiscardingCheckpointsAtShutDown() throws Exception {
	final SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	final Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());

	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final ZooKeeperCompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(client);

	try {
		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);

		checkpointStore.addCheckpoint(checkpoint1);
		assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));

		checkpointStore.shutdown(JobStatus.FINISHED);

		// verify that the checkpoint is discarded
		CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
	} finally {
		client.close();
	}
}
 
Example 6
/**
 * Tests that shutdown discards all checkpoints.
 */
@Test
public void testShutdownDiscardsCheckpoints() throws Exception {
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	CompletedCheckpointStore store = createCompletedCheckpoints(1);
	TestCompletedCheckpoint checkpoint = createCheckpoint(0, sharedStateRegistry);
	Collection<OperatorState> operatorStates = checkpoint.getOperatorStates().values();

	store.addCheckpoint(checkpoint);
	assertEquals(1, store.getNumberOfRetainedCheckpoints());
	verifyCheckpointRegistered(operatorStates, sharedStateRegistry);

	store.shutdown(JobStatus.FINISHED);
	assertEquals(0, store.getNumberOfRetainedCheckpoints());
	assertTrue(checkpoint.isDiscarded());
	verifyCheckpointDiscarded(operatorStates);
}
 
Example 7
/**
 * Tests that suspends discards all checkpoints (as they cannot be
 * recovered later in standalone recovery mode).
 */
@Test
public void testSuspendDiscardsCheckpoints() throws Exception {
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	CompletedCheckpointStore store = createCompletedCheckpoints(1);
	TestCompletedCheckpoint checkpoint = createCheckpoint(0, sharedStateRegistry);
	Collection<OperatorState> taskStates = checkpoint.getOperatorStates().values();

	store.addCheckpoint(checkpoint);
	assertEquals(1, store.getNumberOfRetainedCheckpoints());
	verifyCheckpointRegistered(taskStates, sharedStateRegistry);

	store.shutdown(JobStatus.SUSPENDED);
	assertEquals(0, store.getNumberOfRetainedCheckpoints());
	assertTrue(checkpoint.isDiscarded());
	verifyCheckpointDiscarded(taskStates);
}
 
Example 8
/**
 * Tests that checkpoints are discarded when the completed checkpoint store is shut
 * down with a globally terminal state.
 */
@Test
public void testDiscardingCheckpointsAtShutDown() throws Exception {
	final SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	final Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());

	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final ZooKeeperCompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(client);

	try {
		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);

		checkpointStore.addCheckpoint(checkpoint1);
		assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));

		checkpointStore.shutdown(JobStatus.FINISHED);

		// verify that the checkpoint is discarded
		CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
	} finally {
		client.close();
	}
}
 
Example 9
Source Project: flink   Source File: CompletedCheckpointTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRegisterStatesAtRegistry() {
	OperatorState state = mock(OperatorState.class);
	Map<OperatorID, OperatorState> operatorStates = new HashMap<>();
	operatorStates.put(new OperatorID(), state);

	CompletedCheckpoint checkpoint = new CompletedCheckpoint(
			new JobID(), 0, 0, 1,
			operatorStates,
			Collections.emptyList(),
			CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.RETAIN_ON_FAILURE),
			new TestCompletedCheckpointStorageLocation());

	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	checkpoint.registerSharedStatesAfterRestored(sharedStateRegistry);
	verify(state, times(1)).registerSharedStates(sharedStateRegistry);
}
 
Example 10
Source Project: flink   Source File: CompletedCheckpointStoreTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests adding and getting a checkpoint.
 */
@Test
public void testAddAndGetLatestCheckpoint() throws Exception {
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	CompletedCheckpointStore checkpoints = createCompletedCheckpoints(4);
	
	// Empty state
	assertEquals(0, checkpoints.getNumberOfRetainedCheckpoints());
	assertEquals(0, checkpoints.getAllCheckpoints().size());

	TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] {
			createCheckpoint(0, sharedStateRegistry), createCheckpoint(1, sharedStateRegistry) };

	// Add and get latest
	checkpoints.addCheckpoint(expected[0]);
	assertEquals(1, checkpoints.getNumberOfRetainedCheckpoints());
	verifyCheckpoint(expected[0], checkpoints.getLatestCheckpoint(false));

	checkpoints.addCheckpoint(expected[1]);
	assertEquals(2, checkpoints.getNumberOfRetainedCheckpoints());
	verifyCheckpoint(expected[1], checkpoints.getLatestCheckpoint(false));
}
 
Example 11
private static CheckpointCoordinator instantiateCheckpointCoordinator(JobID jid, ExecutionVertex... ackVertices) {
	return new CheckpointCoordinator(
			jid,
			10000000L,
			600000L,
			0L,
			1,
			CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
			new ExecutionVertex[0],
			ackVertices,
			new ExecutionVertex[0],
			new StandaloneCheckpointIDCounter(),
			new StandaloneCompletedCheckpointStore(10),
			new MemoryStateBackend(),
			Executors.directExecutor(),
			SharedStateRegistry.DEFAULT_FACTORY);
}
 
Example 12
/**
 * Tests that shutdown discards all checkpoints.
 */
@Test
public void testShutdownDiscardsCheckpoints() throws Exception {
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	CompletedCheckpointStore store = createCompletedCheckpoints(1);
	TestCompletedCheckpoint checkpoint = createCheckpoint(0, sharedStateRegistry);
	Collection<OperatorState> operatorStates = checkpoint.getOperatorStates().values();

	store.addCheckpoint(checkpoint);
	assertEquals(1, store.getNumberOfRetainedCheckpoints());
	verifyCheckpointRegistered(operatorStates, sharedStateRegistry);

	store.shutdown(JobStatus.FINISHED);
	assertEquals(0, store.getNumberOfRetainedCheckpoints());
	assertTrue(checkpoint.isDiscarded());
	verifyCheckpointDiscarded(operatorStates);
}
 
Example 13
Source Project: flink   Source File: CompletedCheckpointStoreTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests adding and getting a checkpoint.
 */
@Test
public void testAddAndGetLatestCheckpoint() throws Exception {
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	CompletedCheckpointStore checkpoints = createCompletedCheckpoints(4);
	
	// Empty state
	assertEquals(0, checkpoints.getNumberOfRetainedCheckpoints());
	assertEquals(0, checkpoints.getAllCheckpoints().size());

	TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] {
			createCheckpoint(0, sharedStateRegistry), createCheckpoint(1, sharedStateRegistry) };

	// Add and get latest
	checkpoints.addCheckpoint(expected[0]);
	assertEquals(1, checkpoints.getNumberOfRetainedCheckpoints());
	verifyCheckpoint(expected[0], checkpoints.getLatestCheckpoint(false));

	checkpoints.addCheckpoint(expected[1]);
	assertEquals(2, checkpoints.getNumberOfRetainedCheckpoints());
	verifyCheckpoint(expected[1], checkpoints.getLatestCheckpoint(false));
}
 
Example 14
Source Project: flink   Source File: CompletedCheckpointStoreTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that adding more checkpoints than retained discards the correct checkpoints (using
 * the correct class loader).
 */
@Test
public void testAddCheckpointMoreThanMaxRetained() throws Exception {
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	CompletedCheckpointStore checkpoints = createCompletedCheckpoints(1);

	TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] {
			createCheckpoint(0, sharedStateRegistry), createCheckpoint(1, sharedStateRegistry),
			createCheckpoint(2, sharedStateRegistry), createCheckpoint(3, sharedStateRegistry)
	};

	// Add checkpoints
	checkpoints.addCheckpoint(expected[0]);
	assertEquals(1, checkpoints.getNumberOfRetainedCheckpoints());

	for (int i = 1; i < expected.length; i++) {
		Collection<OperatorState> taskStates = expected[i - 1].getOperatorStates().values();

		checkpoints.addCheckpoint(expected[i]);

		// The ZooKeeper implementation discards asynchronously
		expected[i - 1].awaitDiscard();
		assertTrue(expected[i - 1].isDiscarded());
		assertEquals(1, checkpoints.getNumberOfRetainedCheckpoints());
	}
}
 
Example 15
Source Project: flink   Source File: CompletedCheckpointStoreTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that all added checkpoints are returned.
 */
@Test
public void testGetAllCheckpoints() throws Exception {
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	CompletedCheckpointStore checkpoints = createCompletedCheckpoints(4);

	TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] {
			createCheckpoint(0, sharedStateRegistry), createCheckpoint(1, sharedStateRegistry),
			createCheckpoint(2, sharedStateRegistry), createCheckpoint(3, sharedStateRegistry)
	};

	for (TestCompletedCheckpoint checkpoint : expected) {
		checkpoints.addCheckpoint(checkpoint);
	}

	List<CompletedCheckpoint> actual = checkpoints.getAllCheckpoints();

	assertEquals(expected.length, actual.size());

	for (int i = 0; i < expected.length; i++) {
		assertEquals(expected[i], actual.get(i));
	}
}
 
Example 16
private static CheckpointCoordinator instantiateCheckpointCoordinator(JobID jid, ExecutionVertex... ackVertices) {
	CheckpointCoordinatorConfiguration chkConfig = new CheckpointCoordinatorConfiguration(
		10000000L,
		600000L,
		0L,
		1,
		CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
		true,
		false,
		0);
	return new CheckpointCoordinator(
			jid,
			chkConfig,
			new ExecutionVertex[0],
			ackVertices,
			new ExecutionVertex[0],
			new StandaloneCheckpointIDCounter(),
			new StandaloneCompletedCheckpointStore(10),
			new MemoryStateBackend(),
			Executors.directExecutor(),
			SharedStateRegistry.DEFAULT_FACTORY,
			new CheckpointFailureManager(
				0,
				NoOpFailJobCall.INSTANCE));
}
 
Example 17
/**
 * Tests that checkpoints are discarded when the completed checkpoint store is shut
 * down with a globally terminal state.
 */
@Test
public void testDiscardingCheckpointsAtShutDown() throws Exception {
	final SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	final Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());

	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final ZooKeeperCompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(client);

	try {
		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);

		checkpointStore.addCheckpoint(checkpoint1);
		assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));

		checkpointStore.shutdown(JobStatus.FINISHED);

		// verify that the checkpoint is discarded
		CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
	} finally {
		client.close();
	}
}
 
Example 18
/**
 * Tests that shutdown discards all checkpoints.
 */
@Test
public void testShutdownDiscardsCheckpoints() throws Exception {
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	CompletedCheckpointStore store = createCompletedCheckpoints(1);
	TestCompletedCheckpoint checkpoint = createCheckpoint(0, sharedStateRegistry);
	Collection<OperatorState> operatorStates = checkpoint.getOperatorStates().values();

	store.addCheckpoint(checkpoint);
	assertEquals(1, store.getNumberOfRetainedCheckpoints());
	verifyCheckpointRegistered(operatorStates, sharedStateRegistry);

	store.shutdown(JobStatus.FINISHED);
	assertEquals(0, store.getNumberOfRetainedCheckpoints());
	assertTrue(checkpoint.isDiscarded());
	verifyCheckpointDiscarded(operatorStates);
}
 
Example 19
/**
 * Tests that suspends discards all checkpoints (as they cannot be
 * recovered later in standalone recovery mode).
 */
@Test
public void testSuspendDiscardsCheckpoints() throws Exception {
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	CompletedCheckpointStore store = createCompletedCheckpoints(1);
	TestCompletedCheckpoint checkpoint = createCheckpoint(0, sharedStateRegistry);
	Collection<OperatorState> taskStates = checkpoint.getOperatorStates().values();

	store.addCheckpoint(checkpoint);
	assertEquals(1, store.getNumberOfRetainedCheckpoints());
	verifyCheckpointRegistered(taskStates, sharedStateRegistry);

	store.shutdown(JobStatus.SUSPENDED);
	assertEquals(0, store.getNumberOfRetainedCheckpoints());
	assertTrue(checkpoint.isDiscarded());
	verifyCheckpointDiscarded(taskStates);
}
 
Example 20
/**
 * Tests that shutdown discards all checkpoints.
 */
@Test
public void testShutdownDiscardsCheckpoints() throws Exception {
	CuratorFramework client = ZOOKEEPER.getClient();

	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	CompletedCheckpointStore store = createCompletedCheckpoints(1);
	TestCompletedCheckpoint checkpoint = createCheckpoint(0, sharedStateRegistry);

	store.addCheckpoint(checkpoint);
	assertEquals(1, store.getNumberOfRetainedCheckpoints());
	assertNotNull(client.checkExists().forPath(CHECKPOINT_PATH + ZooKeeperCompletedCheckpointStore.checkpointIdToPath(checkpoint.getCheckpointID())));

	store.shutdown(JobStatus.FINISHED);
	assertEquals(0, store.getNumberOfRetainedCheckpoints());
	assertNull(client.checkExists().forPath(CHECKPOINT_PATH + ZooKeeperCompletedCheckpointStore.checkpointIdToPath(checkpoint.getCheckpointID())));

	sharedStateRegistry.close();
	store.recover();

	assertEquals(0, store.getNumberOfRetainedCheckpoints());
}
 
Example 21
/**
 * FLINK-6284
 *
 * Tests that the latest recovered checkpoint is the one with the highest checkpoint id
 */
@Test
public void testLatestCheckpointRecovery() throws Exception {
	final int numCheckpoints = 3;
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	CompletedCheckpointStore checkpointStore = createCompletedCheckpoints(numCheckpoints);
	List<CompletedCheckpoint> checkpoints = new ArrayList<>(numCheckpoints);

	checkpoints.add(createCheckpoint(9, sharedStateRegistry));
	checkpoints.add(createCheckpoint(10, sharedStateRegistry));
	checkpoints.add(createCheckpoint(11, sharedStateRegistry));

	for (CompletedCheckpoint checkpoint : checkpoints) {
		checkpointStore.addCheckpoint(checkpoint);
	}

	sharedStateRegistry.close();
	checkpointStore.recover();

	CompletedCheckpoint latestCheckpoint = checkpointStore.getLatestCheckpoint(false);

	assertEquals(checkpoints.get(checkpoints.size() -1), latestCheckpoint);
}
 
Example 22
/**
 * FLINK-6284
 *
 * Tests that the latest recovered checkpoint is the one with the highest checkpoint id
 */
@Test
public void testLatestCheckpointRecovery() throws Exception {
	final int numCheckpoints = 3;
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	CompletedCheckpointStore checkpointStore = createCompletedCheckpoints(numCheckpoints);
	List<CompletedCheckpoint> checkpoints = new ArrayList<>(numCheckpoints);

	checkpoints.add(createCheckpoint(9, sharedStateRegistry));
	checkpoints.add(createCheckpoint(10, sharedStateRegistry));
	checkpoints.add(createCheckpoint(11, sharedStateRegistry));

	for (CompletedCheckpoint checkpoint : checkpoints) {
		checkpointStore.addCheckpoint(checkpoint);
	}

	sharedStateRegistry.close();
	checkpointStore.recover();

	CompletedCheckpoint latestCheckpoint = checkpointStore.getLatestCheckpoint(false);

	assertEquals(checkpoints.get(checkpoints.size() -1), latestCheckpoint);
}
 
Example 23
Source Project: flink   Source File: CompletedCheckpointStoreTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that all added checkpoints are returned.
 */
@Test
public void testGetAllCheckpoints() throws Exception {
	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	CompletedCheckpointStore checkpoints = createCompletedCheckpoints(4);

	TestCompletedCheckpoint[] expected = new TestCompletedCheckpoint[] {
			createCheckpoint(0, sharedStateRegistry), createCheckpoint(1, sharedStateRegistry),
			createCheckpoint(2, sharedStateRegistry), createCheckpoint(3, sharedStateRegistry)
	};

	for (TestCompletedCheckpoint checkpoint : expected) {
		checkpoints.addCheckpoint(checkpoint);
	}

	List<CompletedCheckpoint> actual = checkpoints.getAllCheckpoints();

	assertEquals(expected.length, actual.size());

	for (int i = 0; i < expected.length; i++) {
		assertEquals(expected[i], actual.get(i));
	}
}
 
Example 24
Source Project: Flink-CEPplus   Source File: SubtaskState.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void registerSharedStates(SharedStateRegistry sharedStateRegistry) {
	if (managedKeyedState != null) {
		managedKeyedState.registerSharedStates(sharedStateRegistry);
	}

	if (rawKeyedState != null) {
		rawKeyedState.registerSharedStates(sharedStateRegistry);
	}
}
 
Example 25
Source Project: Flink-CEPplus   Source File: OperatorSubtaskState.java    License: Apache License 2.0 5 votes vote down vote up
private static void registerSharedState(
	SharedStateRegistry sharedStateRegistry,
	Iterable<KeyedStateHandle> stateHandles) {
	for (KeyedStateHandle stateHandle : stateHandles) {
		if (stateHandle != null) {
			stateHandle.registerSharedStates(sharedStateRegistry);
		}
	}
}
 
Example 26
Source Project: Flink-CEPplus   Source File: TaskStateSnapshot.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void registerSharedStates(SharedStateRegistry stateRegistry) {
	for (OperatorSubtaskState operatorSubtaskState : subtaskStatesByOperatorID.values()) {
		if (operatorSubtaskState != null) {
			operatorSubtaskState.registerSharedStates(stateRegistry);
		}
	}
}
 
Example 27
/**
 * Tests that subsumed checkpoints are discarded.
 */
@Test
public void testDiscardingSubsumedCheckpoints() throws Exception {
	final SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	final Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());

	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final ZooKeeperCompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(client);

	try {
		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);

		checkpointStore.addCheckpoint(checkpoint1);
		assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));

		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint2 = CompletedCheckpointStoreTest.createCheckpoint(1, sharedStateRegistry);
		checkpointStore.addCheckpoint(checkpoint2);
		final List<CompletedCheckpoint> allCheckpoints = checkpointStore.getAllCheckpoints();
		assertThat(allCheckpoints, Matchers.contains(checkpoint2));
		assertThat(allCheckpoints, Matchers.not(Matchers.contains(checkpoint1)));

		// verify that the subsumed checkpoint is discarded
		CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
	} finally {
		client.close();
	}
}
 
Example 28
protected void verifyCheckpointRegistered(Collection<OperatorState> operatorStates, SharedStateRegistry registry) {
	for (OperatorState operatorState : operatorStates) {
		for (OperatorSubtaskState subtaskState : operatorState.getStates()) {
			Assert.assertTrue(((TestOperatorSubtaskState)subtaskState).registered);
		}
	}
}
 
Example 29
Source Project: flink   Source File: CompletedCheckpointStoreTest.java    License: Apache License 2.0 5 votes vote down vote up
protected void verifyCheckpointRegistered(Collection<OperatorState> operatorStates, SharedStateRegistry registry) {
	for (OperatorState operatorState : operatorStates) {
		for (OperatorSubtaskState subtaskState : operatorState.getStates()) {
			Assert.assertTrue(((TestOperatorSubtaskState)subtaskState).registered);
		}
	}
}
 
Example 30
/**
 * Tests that suspends keeps all checkpoints (so that they can be recovered
 * later by the ZooKeeper store). Furthermore, suspending a job should release
 * all locks.
 */
@Test
public void testSuspendKeepsCheckpoints() throws Exception {
	CuratorFramework client = ZOOKEEPER.getClient();

	SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	CompletedCheckpointStore store = createCompletedCheckpoints(1);
	TestCompletedCheckpoint checkpoint = createCheckpoint(0, sharedStateRegistry);

	store.addCheckpoint(checkpoint);
	assertEquals(1, store.getNumberOfRetainedCheckpoints());
	assertNotNull(client.checkExists().forPath(CHECKPOINT_PATH + ZooKeeperCompletedCheckpointStore.checkpointIdToPath(checkpoint.getCheckpointID())));

	store.shutdown(JobStatus.SUSPENDED);

	assertEquals(0, store.getNumberOfRetainedCheckpoints());

	final String checkpointPath = CHECKPOINT_PATH + ZooKeeperCompletedCheckpointStore.checkpointIdToPath(checkpoint.getCheckpointID());
	Stat stat = client.checkExists().forPath(checkpointPath);

	assertNotNull("The checkpoint node should exist.", stat);
	assertEquals("The checkpoint node should not be locked.", 0, stat.getNumChildren());

	// Recover again
	sharedStateRegistry.close();
	store.recover();

	CompletedCheckpoint recovered = store.getLatestCheckpoint();
	assertEquals(checkpoint, recovered);
}