Java Code Examples for org.apache.flink.api.common.time.Time#hours()

The following examples show how to use org.apache.flink.api.common.time.Time#hours() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Tests that we can cache AccessExecutionGraphs over multiple accesses.
 */
@Test
public void testExecutionGraphCaching() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(expectedJobId, CompletableFuture.completedFuture(expectedExecutionGraph));

	try (ExecutionGraphCache executionGraphCache = new ExecutionGraphCache(timeout, timeToLive)) {
		CompletableFuture<AccessExecutionGraph> accessExecutionGraphFuture = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		assertEquals(expectedExecutionGraph, accessExecutionGraphFuture.get());

		accessExecutionGraphFuture = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		assertEquals(expectedExecutionGraph, accessExecutionGraphFuture.get());

		assertThat(restfulGateway.getNumRequestJobCalls(), Matchers.equalTo(1));
	}
}
 
Example 2
Source Project: flink   File: ExecutionGraphCacheTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that we can cache AccessExecutionGraphs over multiple accesses.
 */
@Test
public void testExecutionGraphCaching() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(expectedJobId, CompletableFuture.completedFuture(expectedExecutionGraph));

	try (ExecutionGraphCache executionGraphCache = new ExecutionGraphCache(timeout, timeToLive)) {
		CompletableFuture<AccessExecutionGraph> accessExecutionGraphFuture = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		assertEquals(expectedExecutionGraph, accessExecutionGraphFuture.get());

		accessExecutionGraphFuture = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		assertEquals(expectedExecutionGraph, accessExecutionGraphFuture.get());

		assertThat(restfulGateway.getNumRequestJobCalls(), Matchers.equalTo(1));
	}
}
 
Example 3
/**
 * Tests that we can cache AccessExecutionGraphs over multiple accesses.
 */
@Test
public void testExecutionGraphCaching() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(expectedJobId, CompletableFuture.completedFuture(expectedExecutionGraph));

	try (ExecutionGraphCache executionGraphCache = new DefaultExecutionGraphCache(timeout, timeToLive)) {
		CompletableFuture<AccessExecutionGraph> accessExecutionGraphFuture = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		assertEquals(expectedExecutionGraph, accessExecutionGraphFuture.get());

		accessExecutionGraphFuture = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		assertEquals(expectedExecutionGraph, accessExecutionGraphFuture.get());

		assertThat(restfulGateway.getNumRequestJobCalls(), Matchers.equalTo(1));
	}
}
 
Example 4
private FileArchivedExecutionGraphStore createDefaultExecutionGraphStore(File storageDirectory) throws IOException {
	return new FileArchivedExecutionGraphStore(
		storageDirectory,
		Time.hours(1L),
		10000L,
		TestingUtils.defaultScheduledExecutor(),
		Ticker.systemTicker());
}
 
Example 5
/**
 * Tests that a failure in requesting an AccessExecutionGraph from the gateway, will not create
 * a cache entry --> another cache request will trigger a new gateway request.
 */
@Test
public void testImmediateCacheInvalidationAfterFailure() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	// let's first answer with a JobNotFoundException and then only with the correct result
	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(
		expectedJobId,
		FutureUtils.completedExceptionally(new FlinkJobNotFoundException(expectedJobId)),
		CompletableFuture.completedFuture(expectedExecutionGraph));

	try (ExecutionGraphCache executionGraphCache = new ExecutionGraphCache(timeout, timeToLive)) {
		CompletableFuture<AccessExecutionGraph> executionGraphFuture = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		try {
			executionGraphFuture.get();

			fail("The execution graph future should have been completed exceptionally.");
		} catch (ExecutionException ee) {
			assertTrue(ee.getCause() instanceof FlinkException);
		}

		CompletableFuture<AccessExecutionGraph> executionGraphFuture2 = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		assertEquals(expectedExecutionGraph, executionGraphFuture2.get());
	}
}
 
Example 6
/**
 * Tests that concurrent accesses only trigger a single AccessExecutionGraph request.
 */
@Test
public void testConcurrentAccess() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(expectedJobId, CompletableFuture.completedFuture(expectedExecutionGraph));

	final int numConcurrentAccesses = 10;

	final ArrayList<CompletableFuture<AccessExecutionGraph>> executionGraphFutures = new ArrayList<>(numConcurrentAccesses);

	final ExecutorService executor = java.util.concurrent.Executors.newFixedThreadPool(numConcurrentAccesses);

	try (ExecutionGraphCache executionGraphCache = new ExecutionGraphCache(timeout, timeToLive)) {
		for (int i = 0; i < numConcurrentAccesses; i++) {
			CompletableFuture<AccessExecutionGraph> executionGraphFuture = CompletableFuture
				.supplyAsync(
					() -> executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway),
					executor)
				.thenCompose(Function.identity());

			executionGraphFutures.add(executionGraphFuture);
		}

		final CompletableFuture<Collection<AccessExecutionGraph>> allExecutionGraphFutures = FutureUtils.combineAll(executionGraphFutures);

		Collection<AccessExecutionGraph> allExecutionGraphs = allExecutionGraphFutures.get();

		for (AccessExecutionGraph executionGraph : allExecutionGraphs) {
			assertEquals(expectedExecutionGraph, executionGraph);
		}

		assertThat(restfulGateway.getNumRequestJobCalls(), Matchers.equalTo(1));
	} finally {
		ExecutorUtils.gracefulShutdown(5000L, TimeUnit.MILLISECONDS, executor);
	}
}
 
Example 7
private FileArchivedExecutionGraphStore createDefaultExecutionGraphStore(File storageDirectory) throws IOException {
	return new FileArchivedExecutionGraphStore(
		storageDirectory,
		Time.hours(1L),
		10000L,
		TestingUtils.defaultScheduledExecutor(),
		Ticker.systemTicker());
}
 
Example 8
Source Project: flink   File: ExecutionGraphCacheTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that a failure in requesting an AccessExecutionGraph from the gateway, will not create
 * a cache entry --> another cache request will trigger a new gateway request.
 */
@Test
public void testImmediateCacheInvalidationAfterFailure() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	// let's first answer with a JobNotFoundException and then only with the correct result
	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(
		expectedJobId,
		FutureUtils.completedExceptionally(new FlinkJobNotFoundException(expectedJobId)),
		CompletableFuture.completedFuture(expectedExecutionGraph));

	try (ExecutionGraphCache executionGraphCache = new ExecutionGraphCache(timeout, timeToLive)) {
		CompletableFuture<AccessExecutionGraph> executionGraphFuture = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		try {
			executionGraphFuture.get();

			fail("The execution graph future should have been completed exceptionally.");
		} catch (ExecutionException ee) {
			assertTrue(ee.getCause() instanceof FlinkException);
		}

		CompletableFuture<AccessExecutionGraph> executionGraphFuture2 = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		assertEquals(expectedExecutionGraph, executionGraphFuture2.get());
	}
}
 
Example 9
Source Project: flink   File: ExecutionGraphCacheTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that concurrent accesses only trigger a single AccessExecutionGraph request.
 */
@Test
public void testConcurrentAccess() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(expectedJobId, CompletableFuture.completedFuture(expectedExecutionGraph));

	final int numConcurrentAccesses = 10;

	final ArrayList<CompletableFuture<AccessExecutionGraph>> executionGraphFutures = new ArrayList<>(numConcurrentAccesses);

	final ExecutorService executor = java.util.concurrent.Executors.newFixedThreadPool(numConcurrentAccesses);

	try (ExecutionGraphCache executionGraphCache = new ExecutionGraphCache(timeout, timeToLive)) {
		for (int i = 0; i < numConcurrentAccesses; i++) {
			CompletableFuture<AccessExecutionGraph> executionGraphFuture = CompletableFuture
				.supplyAsync(
					() -> executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway),
					executor)
				.thenCompose(Function.identity());

			executionGraphFutures.add(executionGraphFuture);
		}

		final CompletableFuture<Collection<AccessExecutionGraph>> allExecutionGraphFutures = FutureUtils.combineAll(executionGraphFutures);

		Collection<AccessExecutionGraph> allExecutionGraphs = allExecutionGraphFutures.get();

		for (AccessExecutionGraph executionGraph : allExecutionGraphs) {
			assertEquals(expectedExecutionGraph, executionGraph);
		}

		assertThat(restfulGateway.getNumRequestJobCalls(), Matchers.equalTo(1));
	} finally {
		ExecutorUtils.gracefulShutdown(5000L, TimeUnit.MILLISECONDS, executor);
	}
}
 
Example 10
private FileArchivedExecutionGraphStore createDefaultExecutionGraphStore(File storageDirectory) throws IOException {
	return new FileArchivedExecutionGraphStore(
		storageDirectory,
		Time.hours(1L),
		Integer.MAX_VALUE,
		10000L,
		TestingUtils.defaultScheduledExecutor(),
		Ticker.systemTicker());
}
 
Example 11
/**
 * Tests that a failure in requesting an AccessExecutionGraph from the gateway, will not create
 * a cache entry --> another cache request will trigger a new gateway request.
 */
@Test
public void testImmediateCacheInvalidationAfterFailure() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	// let's first answer with a JobNotFoundException and then only with the correct result
	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(
		expectedJobId,
		FutureUtils.completedExceptionally(new FlinkJobNotFoundException(expectedJobId)),
		CompletableFuture.completedFuture(expectedExecutionGraph));

	try (ExecutionGraphCache executionGraphCache = new DefaultExecutionGraphCache(timeout, timeToLive)) {
		CompletableFuture<AccessExecutionGraph> executionGraphFuture = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		try {
			executionGraphFuture.get();

			fail("The execution graph future should have been completed exceptionally.");
		} catch (ExecutionException ee) {
			assertTrue(ee.getCause() instanceof FlinkException);
		}

		CompletableFuture<AccessExecutionGraph> executionGraphFuture2 = executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway);

		assertEquals(expectedExecutionGraph, executionGraphFuture2.get());
	}
}
 
Example 12
/**
 * Tests that concurrent accesses only trigger a single AccessExecutionGraph request.
 */
@Test
public void testConcurrentAccess() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(expectedJobId, CompletableFuture.completedFuture(expectedExecutionGraph));

	final int numConcurrentAccesses = 10;

	final ArrayList<CompletableFuture<AccessExecutionGraph>> executionGraphFutures = new ArrayList<>(numConcurrentAccesses);

	final ExecutorService executor = java.util.concurrent.Executors.newFixedThreadPool(numConcurrentAccesses);

	try (ExecutionGraphCache executionGraphCache = new DefaultExecutionGraphCache(timeout, timeToLive)) {
		for (int i = 0; i < numConcurrentAccesses; i++) {
			CompletableFuture<AccessExecutionGraph> executionGraphFuture = CompletableFuture
				.supplyAsync(
					() -> executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway),
					executor)
				.thenCompose(Function.identity());

			executionGraphFutures.add(executionGraphFuture);
		}

		final CompletableFuture<Collection<AccessExecutionGraph>> allExecutionGraphFutures = FutureUtils.combineAll(executionGraphFutures);

		Collection<AccessExecutionGraph> allExecutionGraphs = allExecutionGraphFutures.get();

		for (AccessExecutionGraph executionGraph : allExecutionGraphs) {
			assertEquals(expectedExecutionGraph, executionGraph);
		}

		assertThat(restfulGateway.getNumRequestJobCalls(), Matchers.equalTo(1));
	} finally {
		ExecutorUtils.gracefulShutdown(5000L, TimeUnit.MILLISECONDS, executor);
	}
}
 
Example 13
@Test
public void testSetTtlTimeProvider() throws Exception {
	AbstractStreamOperator<Integer> operator = new AbstractStreamOperator<Integer>() {};
	try (AbstractStreamOperatorTestHarness<Integer> result = new AbstractStreamOperatorTestHarness<>(
			operator,
			1,
			1,
			0)) {

		result.config.setStateKeySerializer(IntSerializer.INSTANCE);

		Time timeToLive = Time.hours(1);
		result.initializeState(new OperatorSubtaskState());
		result.open();

		ValueStateDescriptor<Integer> stateDescriptor = new ValueStateDescriptor<>("test", IntSerializer.INSTANCE);
		stateDescriptor.enableTimeToLive(StateTtlConfig.newBuilder(timeToLive).build());
		KeyedStateBackend<Integer> keyedStateBackend = operator.getKeyedStateBackend();
		ValueState<Integer> state = keyedStateBackend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, stateDescriptor);

		int expectedValue = 42;
		keyedStateBackend.setCurrentKey(1);
		result.setStateTtlProcessingTime(0L);
		state.update(expectedValue);
		Assert.assertEquals(expectedValue, (int) state.value());
		result.setStateTtlProcessingTime(timeToLive.toMilliseconds() + 1);
		Assert.assertNull(state.value());
	}
}