Java Code Examples for org.apache.flink.runtime.state.internal.InternalValueState#update()

The following examples show how to use org.apache.flink.runtime.state.internal.InternalValueState#update() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StateBackendTestBase.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * The purpose of this test is to check that parallel snapshots are possible, and work even if a previous snapshot
 * is still running and blocking.
 */
@Test
public void testParallelAsyncSnapshots() throws Exception {
	OneShotLatch blocker = new OneShotLatch();
	OneShotLatch waiter = new OneShotLatch();
	BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
	streamFactory.setWaiterLatch(waiter);
	streamFactory.setBlockerLatch(blocker);
	streamFactory.setAfterNumberInvocations(10);

	final AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {

		if (!backend.supportsAsynchronousSnapshots()) {
			return;
		}

		// insert some data to the backend.
		InternalValueState<Integer, VoidNamespace, Integer> valueState = backend.createInternalState(
			VoidNamespaceSerializer.INSTANCE,
			new ValueStateDescriptor<>("test", IntSerializer.INSTANCE));

		valueState.setCurrentNamespace(VoidNamespace.INSTANCE);

		for (int i = 0; i < 10; ++i) {
			backend.setCurrentKey(i);
			valueState.update(i);
		}

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot1 =
			backend.snapshot(0L, 0L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());

		Thread runner1 = new Thread(snapshot1, "snapshot-1-runner");
		runner1.start();
		// after this call returns, we have a running snapshot-1 that is blocked in IO.
		waiter.await();

		// do some updates in between the snapshots.
		for (int i = 5; i < 15; ++i) {
			backend.setCurrentKey(i);
			valueState.update(i + 1);
		}

		// we don't want to block the second snapshot.
		streamFactory.setWaiterLatch(null);
		streamFactory.setBlockerLatch(null);

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot2 =
			backend.snapshot(1L, 1L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());

		Thread runner2 = new Thread(snapshot2,"snapshot-2-runner");
		runner2.start();
		// snapshot-2 should run and succeed, while snapshot-1 is still running and blocked in IO.
		snapshot2.get();

		// we release the blocking IO so that snapshot-1 can also finish and succeed.
		blocker.trigger();
		snapshot1.get();

	} finally {
		backend.dispose();
	}
}
Example 2
Source File: StateBackendTestBase.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Test
public void testAsyncSnapshotCancellation() throws Exception {
	OneShotLatch blocker = new OneShotLatch();
	OneShotLatch waiter = new OneShotLatch();
	BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
	streamFactory.setWaiterLatch(waiter);
	streamFactory.setBlockerLatch(blocker);
	streamFactory.setAfterNumberInvocations(10);

	final AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {

		if (!backend.supportsAsynchronousSnapshots()) {
			return;
		}

		InternalValueState<Integer, VoidNamespace, Integer> valueState = backend.createInternalState(
				VoidNamespaceSerializer.INSTANCE,
				new ValueStateDescriptor<>("test", IntSerializer.INSTANCE));

		valueState.setCurrentNamespace(VoidNamespace.INSTANCE);

		for (int i = 0; i < 10; ++i) {
			backend.setCurrentKey(i);
			valueState.update(i);
		}

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot =
				backend.snapshot(0L, 0L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());

		Thread runner = new Thread(snapshot);
		runner.start();

		// wait until the code reached some stream read
		waiter.await();

		// close the backend to see if the close is propagated to the stream
		IOUtils.closeQuietly(backend);

		//unblock the stream so that it can run into the IOException
		blocker.trigger();

		runner.join();

		try {
			snapshot.get();
			fail("Close was not propagated.");
		} catch (CancellationException ex) {
			//ignore
		}

	} finally {
		backend.dispose();
	}
}
Example 3
Source File: StateBackendTestBase.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * The purpose of this test is to check that parallel snapshots are possible, and work even if a previous snapshot
 * is still running and blocking.
 */
@Test
public void testParallelAsyncSnapshots() throws Exception {
	OneShotLatch blocker = new OneShotLatch();
	OneShotLatch waiter = new OneShotLatch();
	BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
	streamFactory.setWaiterLatch(waiter);
	streamFactory.setBlockerLatch(blocker);
	streamFactory.setAfterNumberInvocations(10);

	final AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {

		if (!backend.supportsAsynchronousSnapshots()) {
			return;
		}

		// insert some data to the backend.
		InternalValueState<Integer, VoidNamespace, Integer> valueState = backend.createInternalState(
			VoidNamespaceSerializer.INSTANCE,
			new ValueStateDescriptor<>("test", IntSerializer.INSTANCE));

		valueState.setCurrentNamespace(VoidNamespace.INSTANCE);

		for (int i = 0; i < 10; ++i) {
			backend.setCurrentKey(i);
			valueState.update(i);
		}

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot1 =
			backend.snapshot(0L, 0L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());

		Thread runner1 = new Thread(snapshot1, "snapshot-1-runner");
		runner1.start();
		// after this call returns, we have a running snapshot-1 that is blocked in IO.
		waiter.await();

		// do some updates in between the snapshots.
		for (int i = 5; i < 15; ++i) {
			backend.setCurrentKey(i);
			valueState.update(i + 1);
		}

		// we don't want to block the second snapshot.
		streamFactory.setWaiterLatch(null);
		streamFactory.setBlockerLatch(null);

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot2 =
			backend.snapshot(1L, 1L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());

		Thread runner2 = new Thread(snapshot2,"snapshot-2-runner");
		runner2.start();
		// snapshot-2 should run and succeed, while snapshot-1 is still running and blocked in IO.
		snapshot2.get();

		// we release the blocking IO so that snapshot-1 can also finish and succeed.
		blocker.trigger();
		snapshot1.get();

	} finally {
		backend.dispose();
	}
}
Example 4
Source File: StateBackendTestBase.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testAsyncSnapshotCancellation() throws Exception {
	OneShotLatch blocker = new OneShotLatch();
	OneShotLatch waiter = new OneShotLatch();
	BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
	streamFactory.setWaiterLatch(waiter);
	streamFactory.setBlockerLatch(blocker);
	streamFactory.setAfterNumberInvocations(10);

	final AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {

		if (!backend.supportsAsynchronousSnapshots()) {
			return;
		}

		InternalValueState<Integer, VoidNamespace, Integer> valueState = backend.createInternalState(
				VoidNamespaceSerializer.INSTANCE,
				new ValueStateDescriptor<>("test", IntSerializer.INSTANCE));

		valueState.setCurrentNamespace(VoidNamespace.INSTANCE);

		for (int i = 0; i < 10; ++i) {
			backend.setCurrentKey(i);
			valueState.update(i);
		}

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot =
				backend.snapshot(0L, 0L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());

		Thread runner = new Thread(snapshot);
		runner.start();

		// wait until the code reached some stream read
		waiter.await();

		// close the backend to see if the close is propagated to the stream
		IOUtils.closeQuietly(backend);

		//unblock the stream so that it can run into the IOException
		blocker.trigger();

		runner.join();

		try {
			snapshot.get();
			fail("Close was not propagated.");
		} catch (CancellationException ex) {
			//ignore
		}

	} finally {
		backend.dispose();
	}
}
Example 5
Source File: StateBackendTestBase.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * The purpose of this test is to check that parallel snapshots are possible, and work even if a previous snapshot
 * is still running and blocking.
 */
@Test
public void testParallelAsyncSnapshots() throws Exception {
	OneShotLatch blocker = new OneShotLatch();
	OneShotLatch waiter = new OneShotLatch();
	BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
	streamFactory.setWaiterLatch(waiter);
	streamFactory.setBlockerLatch(blocker);
	streamFactory.setAfterNumberInvocations(10);

	final AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {

		if (!backend.supportsAsynchronousSnapshots()) {
			return;
		}

		// insert some data to the backend.
		InternalValueState<Integer, VoidNamespace, Integer> valueState = backend.createInternalState(
			VoidNamespaceSerializer.INSTANCE,
			new ValueStateDescriptor<>("test", IntSerializer.INSTANCE));

		valueState.setCurrentNamespace(VoidNamespace.INSTANCE);

		for (int i = 0; i < 10; ++i) {
			backend.setCurrentKey(i);
			valueState.update(i);
		}

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot1 =
			backend.snapshot(0L, 0L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());

		Thread runner1 = new Thread(snapshot1, "snapshot-1-runner");
		runner1.start();
		// after this call returns, we have a running snapshot-1 that is blocked in IO.
		waiter.await();

		// do some updates in between the snapshots.
		for (int i = 5; i < 15; ++i) {
			backend.setCurrentKey(i);
			valueState.update(i + 1);
		}

		// we don't want to block the second snapshot.
		streamFactory.setWaiterLatch(null);
		streamFactory.setBlockerLatch(null);

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot2 =
			backend.snapshot(1L, 1L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());

		Thread runner2 = new Thread(snapshot2,"snapshot-2-runner");
		runner2.start();
		// snapshot-2 should run and succeed, while snapshot-1 is still running and blocked in IO.
		snapshot2.get();

		// we release the blocking IO so that snapshot-1 can also finish and succeed.
		blocker.trigger();
		snapshot1.get();

	} finally {
		backend.dispose();
	}
}
Example 6
Source File: StateBackendTestBase.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testAsyncSnapshotCancellation() throws Exception {
	OneShotLatch blocker = new OneShotLatch();
	OneShotLatch waiter = new OneShotLatch();
	BlockerCheckpointStreamFactory streamFactory = new BlockerCheckpointStreamFactory(1024 * 1024);
	streamFactory.setWaiterLatch(waiter);
	streamFactory.setBlockerLatch(blocker);
	streamFactory.setAfterNumberInvocations(10);

	final AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);

	try {

		if (!backend.supportsAsynchronousSnapshots()) {
			return;
		}

		InternalValueState<Integer, VoidNamespace, Integer> valueState = backend.createInternalState(
				VoidNamespaceSerializer.INSTANCE,
				new ValueStateDescriptor<>("test", IntSerializer.INSTANCE));

		valueState.setCurrentNamespace(VoidNamespace.INSTANCE);

		for (int i = 0; i < 10; ++i) {
			backend.setCurrentKey(i);
			valueState.update(i);
		}

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot =
				backend.snapshot(0L, 0L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());

		Thread runner = new Thread(snapshot);
		runner.start();

		// wait until the code reached some stream read
		waiter.await();

		// close the backend to see if the close is propagated to the stream
		IOUtils.closeQuietly(backend);

		//unblock the stream so that it can run into the IOException
		blocker.trigger();

		runner.join();

		try {
			snapshot.get();
			fail("Close was not propagated.");
		} catch (CancellationException ex) {
			//ignore
		}

	} finally {
		backend.dispose();
	}
}