Java Code Examples for java.util.concurrent.RunnableFuture#run()

The following examples show how to use java.util.concurrent.RunnableFuture#run() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: JUnitProjectOpenedHook.java From netbeans with Apache License 2.0

6 votes

@Override
public java.util.concurrent.Future<ProjectProblemsProvider.Result> resolve() {
    ProjectProblemsProvider.Result res;
    if (action != null) {
        action.actionPerformed(null);
        String text = (String) action.getValue(ACT_START_MESSAGE);
        if (text != null) {
            res = ProjectProblemsProvider.Result.create(ProjectProblemsProvider.Status.RESOLVED, text);
        } else {
            res = ProjectProblemsProvider.Result.create(ProjectProblemsProvider.Status.RESOLVED);
        }
    } else {
        res = ProjectProblemsProvider.Result.create(ProjectProblemsProvider.Status.UNRESOLVED, "No resolution for the problem");
    }
    RunnableFuture<ProjectProblemsProvider.Result> f = new FutureTask<>(new Runnable() {
        @Override
        public void run() {
        }
    }, res);
    f.run();
    return f;
}

Example 2

Source File: FutureUtils.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Run the given {@code RunnableFuture} if it is not done, and then retrieves its result.
 * @param future to run if not done and get
 * @param <T> type of the result
 * @return the result after running the future
 * @throws ExecutionException if a problem occurred
 * @throws InterruptedException if the current thread has been interrupted
 */
public static <T> T runIfNotDoneAndGet(RunnableFuture<T> future) throws ExecutionException, InterruptedException {

	if (null == future) {
		return null;
	}

	if (!future.isDone()) {
		future.run();
	}

	return future.get();
}

Example 3

Source File: StateBackendTestBase.java From Flink-CEPplus with Apache License 2.0

5 votes

protected KeyedStateHandle runSnapshot(
	RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotRunnableFuture,
	SharedStateRegistry sharedStateRegistry) throws Exception {

	if (!snapshotRunnableFuture.isDone()) {
		snapshotRunnableFuture.run();
	}

	SnapshotResult<KeyedStateHandle> snapshotResult = snapshotRunnableFuture.get();
	KeyedStateHandle jobManagerOwnedSnapshot = snapshotResult.getJobManagerOwnedSnapshot();
	if (jobManagerOwnedSnapshot != null) {
		jobManagerOwnedSnapshot.registerSharedStates(sharedStateRegistry);
	}
	return jobManagerOwnedSnapshot;
}

Example 4

Source File: ListenableRunnableFutureInterfaceTest.java From threadly with Mozilla Public License 2.0

5 votes

@Test
public void getRunnableResultTest() throws InterruptedException, ExecutionException {
  ExecuteOnGetFutureFactory ff = makeFutureFactory();
  final Object result = new Object();
  RunnableFuture<Object> future = ff.make(DoNothingRunnable.instance(), result);
  
  future.run();
  
  assertTrue(future.get() == result);
}

Example 5

Source File: ListenableRunnableFutureInterfaceTest.java From threadly with Mozilla Public License 2.0

5 votes

@Test
public void getCallableResultTest() throws InterruptedException, ExecutionException {
  ExecuteOnGetFutureFactory ff = makeFutureFactory();
  final Object result = new Object();
  RunnableFuture<Object> future = ff.make(new Callable<Object>() {
    @Override
    public Object call() {
      return result;
    }
  });
  
  future.run();
  
  assertTrue(future.get() == result);
}

Example 6

Source File: StateBackendTestContext.java From Flink-CEPplus with Apache License 2.0

5 votes

@Nonnull
RunnableFuture<SnapshotResult<KeyedStateHandle>> triggerSnapshot() throws Exception {
	RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotRunnableFuture =
		keyedStateBackend.snapshot(682375462392L, 10L,
			checkpointStorageLocation, CheckpointOptions.forCheckpointWithDefaultLocation());
	if (!snapshotRunnableFuture.isDone()) {
		snapshotRunnableFuture.run();
	}
	return snapshotRunnableFuture;
}

Example 7

Source File: FutureUtils.java From flink with Apache License 2.0

5 votes

/**
 * Run the given {@code RunnableFuture} if it is not done, and then retrieves its result.
 * @param future to run if not done and get
 * @param <T> type of the result
 * @return the result after running the future
 * @throws ExecutionException if a problem occurred
 * @throws InterruptedException if the current thread has been interrupted
 */
public static <T> T runIfNotDoneAndGet(RunnableFuture<T> future) throws ExecutionException, InterruptedException {

	if (null == future) {
		return null;
	}

	if (!future.isDone()) {
		future.run();
	}

	return future.get();
}

Example 8

Source File: ListenableRunnableFutureInterfaceTest.java From threadly with Mozilla Public License 2.0

5 votes

@Test
public void isDoneFail() {
  ExecuteOnGetFutureFactory ff = makeFutureFactory();
  TestRunnable r = new TestRuntimeFailureRunnable();
  RunnableFuture<?> future = ff.make(r);
  
  future.run();
  
  assertTrue(future.isDone());
}

Example 9

Source File: StateBackendMigrationTestBase.java From flink with Apache License 2.0

5 votes

private OperatorStateHandle runSnapshot(
	RunnableFuture<SnapshotResult<OperatorStateHandle>> snapshotRunnableFuture) throws Exception {

	if (!snapshotRunnableFuture.isDone()) {
		snapshotRunnableFuture.run();
	}

	return snapshotRunnableFuture.get().getJobManagerOwnedSnapshot();
}

Example 10

Source File: StateBackendMigrationTestBase.java From flink with Apache License 2.0

5 votes

private KeyedStateHandle runSnapshot(
	RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotRunnableFuture,
	SharedStateRegistry sharedStateRegistry) throws Exception {

	if (!snapshotRunnableFuture.isDone()) {
		snapshotRunnableFuture.run();
	}

	SnapshotResult<KeyedStateHandle> snapshotResult = snapshotRunnableFuture.get();
	KeyedStateHandle jobManagerOwnedSnapshot = snapshotResult.getJobManagerOwnedSnapshot();
	if (jobManagerOwnedSnapshot != null) {
		jobManagerOwnedSnapshot.registerSharedStates(sharedStateRegistry);
	}
	return jobManagerOwnedSnapshot;
}

Example 11

Source File: StateBackendMigrationTestBase.java From flink with Apache License 2.0

5 votes

private OperatorStateHandle runSnapshot(
	RunnableFuture<SnapshotResult<OperatorStateHandle>> snapshotRunnableFuture) throws Exception {

	if (!snapshotRunnableFuture.isDone()) {
		snapshotRunnableFuture.run();
	}

	return snapshotRunnableFuture.get().getJobManagerOwnedSnapshot();
}

Example 12

Source File: ListenableRunnableFutureInterfaceTest.java From threadly with Mozilla Public License 2.0

5 votes

@Test
public void isDoneTest() {
  ExecuteOnGetFutureFactory ff = makeFutureFactory();
  TestRunnable r = new TestRunnable();
  RunnableFuture<?> future = ff.make(r);
  future.run();
  
  assertTrue(future.isDone());
}

Example 13

Source File: HeapKeyedStateBackendSnapshotMigrationTest.java From flink with Apache License 2.0

4 votes

@Test
public void testMapStateMigrationAfterHashMapSerRemoval() throws Exception {
	ClassLoader cl = getClass().getClassLoader();
	URL resource = cl.getResource("heap_keyed_statebackend_1_5_map.snapshot");

	Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");

	final SnapshotResult<KeyedStateHandle> stateHandles;
	try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
		stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
	}
	final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
	try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
		final Integer namespace1 = 1;
		final Integer namespace2 = 2;
		final Integer namespace3 = 3;

		final MapStateDescriptor<Long, Long> stateDescr = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
		stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());

		InternalMapState<String, Integer, Long, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);

		keyedBackend.setCurrentKey("abc");
		state.setCurrentNamespace(namespace1);
		assertEquals(33L, (long) state.get(33L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace2);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(11L, (long) state.get(11L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(44L, (long) state.get(44L));
		assertEquals(1, getStateSize(state));

		keyedBackend.setCurrentKey("def");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(3, getStateSize(state));

		keyedBackend.setCurrentKey("jkl");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		keyedBackend.setCurrentKey("mno");
		state.setCurrentNamespace(namespace3);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedBackend.snapshot(
				1L,
				1L,
				new MemCheckpointStreamFactory(4 * 1024 * 1024),
				CheckpointOptions.forCheckpointWithDefaultLocation());

		snapshot.run();
	}
}

Example 14

Source File: RocksDBStateBackendTest.java From flink with Apache License 2.0

4 votes

@Test
public void testSharedIncrementalStateDeRegistration() throws Exception {
	if (enableIncrementalCheckpointing) {
		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
		try {
			ValueStateDescriptor<String> kvId =
				new ValueStateDescriptor<>("id", String.class, null);

			kvId.initializeSerializerUnlessSet(new ExecutionConfig());

			ValueState<String> state =
				backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

			Queue<IncrementalRemoteKeyedStateHandle> previousStateHandles = new LinkedList<>();
			SharedStateRegistry sharedStateRegistry = spy(new SharedStateRegistry());
			for (int checkpointId = 0; checkpointId < 3; ++checkpointId) {

				reset(sharedStateRegistry);

				backend.setCurrentKey(checkpointId);
				state.update("Hello-" + checkpointId);

				RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = backend.snapshot(
					checkpointId,
					checkpointId,
					createStreamFactory(),
					CheckpointOptions.forCheckpointWithDefaultLocation());

				snapshot.run();

				SnapshotResult<KeyedStateHandle> snapshotResult = snapshot.get();

				IncrementalRemoteKeyedStateHandle stateHandle =
					(IncrementalRemoteKeyedStateHandle) snapshotResult.getJobManagerOwnedSnapshot();

				Map<StateHandleID, StreamStateHandle> sharedState =
					new HashMap<>(stateHandle.getSharedState());

				stateHandle.registerSharedStates(sharedStateRegistry);

				for (Map.Entry<StateHandleID, StreamStateHandle> e : sharedState.entrySet()) {
					verify(sharedStateRegistry).registerReference(
						stateHandle.createSharedStateRegistryKeyFromFileName(e.getKey()),
						e.getValue());
				}

				previousStateHandles.add(stateHandle);
				backend.notifyCheckpointComplete(checkpointId);

				//-----------------------------------------------------------------

				if (previousStateHandles.size() > 1) {
					checkRemove(previousStateHandles.remove(), sharedStateRegistry);
				}
			}

			while (!previousStateHandles.isEmpty()) {

				reset(sharedStateRegistry);

				checkRemove(previousStateHandles.remove(), sharedStateRegistry);
			}
		} finally {
			IOUtils.closeQuietly(backend);
			backend.dispose();
		}
	}
}

Example 15

Source File: DefaultClassPathProviderTest.java From netbeans with Apache License 2.0

4 votes

/**
 * Checks that single event is fired during opening (closing) of projects
 */
public void testEvents() throws Exception {
    final File wd = getWorkDir();
    final File root1 = FileUtil.normalizeFile(new File (wd,"src1"));    //NOI18N
    final File root2 = FileUtil.normalizeFile(new File (wd,"src2"));    //NOI18N
    root1.mkdir();
    root2.mkdir();
    final ClassPathProvider cpp = new DefaultClassPathProvider ();
    final ClassPath defaultCompile = cpp.findClassPath(FileUtil.toFileObject(wd), ClassPath.COMPILE);
    assertNotNull(defaultCompile);
    assertFalse(contains(defaultCompile, Utilities.toURI(root1).toURL()));
    assertFalse(contains(defaultCompile, Utilities.toURI(root2).toURL()));
    final Listener listener = new Listener();
    defaultCompile.addPropertyChangeListener(listener);
    final GlobalPathRegistry regs = GlobalPathRegistry.getDefault();
    final ClassPath compileCpProject1 = ClassPathSupport.createClassPath(Utilities.toURI(root1).toURL());
    final ClassPath compileCpProject2 = ClassPathSupport.createClassPath(Utilities.toURI(root2).toURL());
    
    //Simulate projects open
    RunnableFuture<Project[]> barrier = new FutureTask<Project[]>(new Callable<Project[]>() {
        @Override
        public Project[] call() throws Exception {
            return new Project[0];
        }
    });
    OpenProject.future = barrier;

    regs.register(ClassPath.COMPILE, new ClassPath[]{compileCpProject1});
    assertFalse(contains(defaultCompile, Utilities.toURI(root1).toURL()));
    assertFalse(listener.awaitEvent());
    assertEquals(0, listener.get());
    regs.register(ClassPath.COMPILE, new ClassPath[]{compileCpProject2});
    assertFalse(contains(defaultCompile, Utilities.toURI(root2).toURL()));
    assertFalse(listener.awaitEvent());
    assertEquals(0, listener.get());
    barrier.run();
    assertTrue(listener.awaitEvent());
    assertEquals(1, listener.get());
    assertTrue(contains(defaultCompile, Utilities.toURI(root1).toURL()));
    assertTrue(contains(defaultCompile, Utilities.toURI(root2).toURL()));

    //Simulate projects close
    barrier = new FutureTask<Project[]>(new Callable<Project[]>() {
        @Override
        public Project[] call() throws Exception {
            return new Project[0];
        }
    });
    OpenProject.future = barrier;
    listener.reset();
    regs.unregister(ClassPath.COMPILE, new ClassPath[]{compileCpProject1});
    assertTrue(contains(defaultCompile, Utilities.toURI(root1).toURL()));
    assertFalse(listener.awaitEvent());
    assertEquals(0, listener.get());
    regs.unregister(ClassPath.COMPILE, new ClassPath[]{compileCpProject2});
    assertTrue(contains(defaultCompile, Utilities.toURI(root2).toURL()));
    assertFalse(listener.awaitEvent());
    assertEquals(0, listener.get());
    barrier.run();
    assertTrue(listener.awaitEvent());
    assertEquals(1, listener.get());
    assertFalse(contains(defaultCompile, Utilities.toURI(root1).toURL()));
    assertFalse(contains(defaultCompile, Utilities.toURI(root2).toURL()));
}

Example 16

Source File: HeapKeyedStateBackendSnapshotMigrationTest.java From flink with Apache License 2.0

4 votes

@Test
public void testMapStateMigrationAfterHashMapSerRemoval() throws Exception {
	ClassLoader cl = getClass().getClassLoader();
	URL resource = cl.getResource("heap_keyed_statebackend_1_5_map.snapshot");

	Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");

	final SnapshotResult<KeyedStateHandle> stateHandles;
	try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
		stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
	}
	final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
	try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
		final Integer namespace1 = 1;
		final Integer namespace2 = 2;
		final Integer namespace3 = 3;

		final MapStateDescriptor<Long, Long> stateDescr = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
		stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());

		InternalMapState<String, Integer, Long, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);

		keyedBackend.setCurrentKey("abc");
		state.setCurrentNamespace(namespace1);
		assertEquals(33L, (long) state.get(33L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace2);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(11L, (long) state.get(11L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(44L, (long) state.get(44L));
		assertEquals(1, getStateSize(state));

		keyedBackend.setCurrentKey("def");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(3, getStateSize(state));

		keyedBackend.setCurrentKey("jkl");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		keyedBackend.setCurrentKey("mno");
		state.setCurrentNamespace(namespace3);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedBackend.snapshot(
				1L,
				1L,
				new MemCheckpointStreamFactory(4 * 1024 * 1024),
				CheckpointOptions.forCheckpointWithDefaultLocation());

		snapshot.run();
	}
}

Example 17

Source File: ProfileProblemsProviderImpl.java From netbeans with Apache License 2.0

4 votes

@Override
@NbBundle.Messages({
    "LBL_ResolveProfile=Resolve Invalid Project Profile",
    "LBL_ResolveButton=OK",
    "AN_ResolveButton=Resolve",
    "AD_ResolveButton=Resolve the profile problems"
})
public Future<Result> resolve() {
    final JButton ok = new JButton(LBL_ResolveButton());
    ok.getAccessibleContext().setAccessibleName(AN_ResolveButton());
    ok.getAccessibleContext().setAccessibleDescription(AD_ResolveButton());
    final FixProfile panel = new FixProfile(ok, currentProfile, state);
    final DialogDescriptor dd = new DialogDescriptor(
            panel,
            LBL_ResolveProfile(),
            true,
            new Object[]{
        ok,
        DialogDescriptor.CANCEL_OPTION
    },
            ok,
            DialogDescriptor.DEFAULT_ALIGN,
            null,
            null);
    if (DialogDisplayer.getDefault().notify(dd) == ok) {
        return RP.submit(new Callable<ProjectProblemsProvider.Result>() {
            @Override
            public ProjectProblemsProvider.Result call() throws Exception {
                ProjectProblemsProvider.Status status = ProjectProblemsProvider.Status.UNRESOLVED;
                try {
                    ProjectManager.mutex().writeAccess(new Mutex.ExceptionAction<Void>() {
                        @Override
                        public Void run() throws IOException {
                            final boolean shouldUpdate = panel.shouldUpdateProfile();
                            if (shouldUpdate) {
                                final Profile newProfile = panel.getProfile();
                                final EditableProperties props = antProjectHelper.getProperties(AntProjectHelper.PROJECT_PROPERTIES_PATH);
                                if (newProfile == null ||
                                    newProfile == Profile.DEFAULT) {
                                    props.remove(profileProperty);
                                } else {
                                    props.put(profileProperty, newProfile.getName());
                                }
                                antProjectHelper.putProperties(AntProjectHelper.PROJECT_PROPERTIES_PATH, props);
                            }
                            for (Reference rootToRemove : panel.getRootsToRemove()) {
                                rootToRemove.remove(antProjectHelper);
                            }
                            ProjectManager.getDefault().saveProject(
                                    FileOwnerQuery.getOwner(antProjectHelper.getProjectDirectory()));
                            return null;
                        }
                    });
                    status = ProjectProblemsProvider.Status.RESOLVED;
                } catch (MutexException e) {
                    Exceptions.printStackTrace(e);
                }
                return ProjectProblemsProvider.Result.create(status);
            }
        });
    }
    final RunnableFuture<Result> res = new FutureTask<Result>(
            new Callable<Result>() {
                @Override
                public Result call() {
                    return ProjectProblemsProvider.Result.create(ProjectProblemsProvider.Status.UNRESOLVED);
                }
            });
    res.run();
    return res;
}

Example 18

Source File: RocksDBStateBackendTest.java From flink with Apache License 2.0

4 votes

@Test
public void testSharedIncrementalStateDeRegistration() throws Exception {
	if (enableIncrementalCheckpointing) {
		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
		try {
			ValueStateDescriptor<String> kvId =
				new ValueStateDescriptor<>("id", String.class, null);

			kvId.initializeSerializerUnlessSet(new ExecutionConfig());

			ValueState<String> state =
				backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

			Queue<IncrementalRemoteKeyedStateHandle> previousStateHandles = new LinkedList<>();
			SharedStateRegistry sharedStateRegistry = spy(new SharedStateRegistry());
			for (int checkpointId = 0; checkpointId < 3; ++checkpointId) {

				reset(sharedStateRegistry);

				backend.setCurrentKey(checkpointId);
				state.update("Hello-" + checkpointId);

				RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = backend.snapshot(
					checkpointId,
					checkpointId,
					createStreamFactory(),
					CheckpointOptions.forCheckpointWithDefaultLocation());

				snapshot.run();

				SnapshotResult<KeyedStateHandle> snapshotResult = snapshot.get();

				IncrementalRemoteKeyedStateHandle stateHandle =
					(IncrementalRemoteKeyedStateHandle) snapshotResult.getJobManagerOwnedSnapshot();

				Map<StateHandleID, StreamStateHandle> sharedState =
					new HashMap<>(stateHandle.getSharedState());

				stateHandle.registerSharedStates(sharedStateRegistry);

				for (Map.Entry<StateHandleID, StreamStateHandle> e : sharedState.entrySet()) {
					verify(sharedStateRegistry).registerReference(
						stateHandle.createSharedStateRegistryKeyFromFileName(e.getKey()),
						e.getValue());
				}

				previousStateHandles.add(stateHandle);
				backend.notifyCheckpointComplete(checkpointId);

				//-----------------------------------------------------------------

				if (previousStateHandles.size() > 1) {
					checkRemove(previousStateHandles.remove(), sharedStateRegistry);
				}
			}

			while (!previousStateHandles.isEmpty()) {

				reset(sharedStateRegistry);

				checkRemove(previousStateHandles.remove(), sharedStateRegistry);
			}
		} finally {
			IOUtils.closeQuietly(backend);
			backend.dispose();
		}
	}
}

Example 19

Source File: HeapKeyedStateBackendSnapshotMigrationTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testMapStateMigrationAfterHashMapSerRemoval() throws Exception {
	ClassLoader cl = getClass().getClassLoader();
	URL resource = cl.getResource("heap_keyed_statebackend_1_5_map.snapshot");

	Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");

	final SnapshotResult<KeyedStateHandle> stateHandles;
	try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
		stateHandles = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
	}
	final KeyedStateHandle stateHandle = stateHandles.getJobManagerOwnedSnapshot();
	try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend(StateObjectCollection.singleton(stateHandle))) {
		final Integer namespace1 = 1;
		final Integer namespace2 = 2;
		final Integer namespace3 = 3;

		final MapStateDescriptor<Long, Long> stateDescr = new MapStateDescriptor<>("my-map-state", Long.class, Long.class);
		stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());

		InternalMapState<String, Integer, Long, Long> state = keyedBackend.createInternalState(IntSerializer.INSTANCE, stateDescr);

		keyedBackend.setCurrentKey("abc");
		state.setCurrentNamespace(namespace1);
		assertEquals(33L, (long) state.get(33L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace2);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(11L, (long) state.get(11L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(44L, (long) state.get(44L));
		assertEquals(1, getStateSize(state));

		keyedBackend.setCurrentKey("def");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(2, getStateSize(state));

		state.setCurrentNamespace(namespace3);
		assertEquals(22L, (long) state.get(22L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(3, getStateSize(state));

		keyedBackend.setCurrentKey("jkl");
		state.setCurrentNamespace(namespace1);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		keyedBackend.setCurrentKey("mno");
		state.setCurrentNamespace(namespace3);
		assertEquals(11L, (long) state.get(11L));
		assertEquals(22L, (long) state.get(22L));
		assertEquals(33L, (long) state.get(33L));
		assertEquals(44L, (long) state.get(44L));
		assertEquals(55L, (long) state.get(55L));
		assertEquals(5, getStateSize(state));

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = keyedBackend.snapshot(
				1L,
				1L,
				new MemCheckpointStreamFactory(4 * 1024 * 1024),
				CheckpointOptions.forCheckpointWithDefaultLocation());

		snapshot.run();
	}
}

Example 20

Source File: RocksDBStateBackendTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testSharedIncrementalStateDeRegistration() throws Exception {
	if (enableIncrementalCheckpointing) {
		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
		try {
			ValueStateDescriptor<String> kvId =
				new ValueStateDescriptor<>("id", String.class, null);

			kvId.initializeSerializerUnlessSet(new ExecutionConfig());

			ValueState<String> state =
				backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

			Queue<IncrementalRemoteKeyedStateHandle> previousStateHandles = new LinkedList<>();
			SharedStateRegistry sharedStateRegistry = spy(new SharedStateRegistry());
			for (int checkpointId = 0; checkpointId < 3; ++checkpointId) {

				reset(sharedStateRegistry);

				backend.setCurrentKey(checkpointId);
				state.update("Hello-" + checkpointId);

				RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = backend.snapshot(
					checkpointId,
					checkpointId,
					createStreamFactory(),
					CheckpointOptions.forCheckpointWithDefaultLocation());

				snapshot.run();

				SnapshotResult<KeyedStateHandle> snapshotResult = snapshot.get();

				IncrementalRemoteKeyedStateHandle stateHandle =
					(IncrementalRemoteKeyedStateHandle) snapshotResult.getJobManagerOwnedSnapshot();

				Map<StateHandleID, StreamStateHandle> sharedState =
					new HashMap<>(stateHandle.getSharedState());

				stateHandle.registerSharedStates(sharedStateRegistry);

				for (Map.Entry<StateHandleID, StreamStateHandle> e : sharedState.entrySet()) {
					verify(sharedStateRegistry).registerReference(
						stateHandle.createSharedStateRegistryKeyFromFileName(e.getKey()),
						e.getValue());
				}

				previousStateHandles.add(stateHandle);
				backend.notifyCheckpointComplete(checkpointId);

				//-----------------------------------------------------------------

				if (previousStateHandles.size() > 1) {
					checkRemove(previousStateHandles.remove(), sharedStateRegistry);
				}
			}

			while (!previousStateHandles.isEmpty()) {

				reset(sharedStateRegistry);

				checkRemove(previousStateHandles.remove(), sharedStateRegistry);
			}
		} finally {
			IOUtils.closeQuietly(backend);
			backend.dispose();
		}
	}
}