org.apache.flink.runtime.concurrent.Executors Java Examples

The following examples show how to use org.apache.flink.runtime.concurrent.Executors. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AggregatingMetricsHandlerTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
	MetricFetcher fetcher = new MetricFetcherImpl<RestfulGateway>(
		mock(GatewayRetriever.class),
		mock(MetricQueryServiceRetriever.class),
		Executors.directExecutor(),
		TestingUtils.TIMEOUT(),
		MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());
	store = fetcher.getMetricStore();

	Collection<MetricDump> metricDumps = getMetricDumps();
	for (MetricDump dump : metricDumps) {
		store.add(dump);
	}

	handler = getHandler(
		LEADER_RETRIEVER,
		TIMEOUT,
		TEST_HEADERS,
		EXECUTOR,
		fetcher);
	pathParameters = getPathParameters();
}
 
Example #2
Source File: RestClientTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testConnectionTimeout() throws Exception {
	final Configuration config = new Configuration();
	config.setLong(RestOptions.CONNECTION_TIMEOUT, 1);
	try (final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(config), Executors.directExecutor())) {
		restClient.sendRequest(
			unroutableIp,
			80,
			new TestMessageHeaders(),
			EmptyMessageParameters.getInstance(),
			EmptyRequestBody.getInstance())
			.get(60, TimeUnit.SECONDS);
	} catch (final ExecutionException e) {
		final Throwable throwable = ExceptionUtils.stripExecutionException(e);
		assertThat(throwable, instanceOf(ConnectTimeoutException.class));
		assertThat(throwable.getMessage(), containsString(unroutableIp));
	}
}
 
Example #3
Source File: RestClientTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testInvalidVersionRejection() throws Exception {
	try (final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(new Configuration()), Executors.directExecutor())) {
		CompletableFuture<EmptyResponseBody> invalidVersionResponse = restClient.sendRequest(
			unroutableIp,
			80,
			new TestMessageHeaders(),
			EmptyMessageParameters.getInstance(),
			EmptyRequestBody.getInstance(),
			Collections.emptyList(),
			RestAPIVersion.V0
		);
		Assert.fail("The request should have been rejected due to a version mismatch.");
	} catch (IllegalArgumentException e) {
		// expected
	}
}
 
Example #4
Source File: ExecutionJobVertexTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private static ExecutionJobVertex createExecutionJobVertex(
		int parallelism,
		int preconfiguredMaxParallelism) throws JobException {

	JobVertex jobVertex = new JobVertex("testVertex");
	jobVertex.setInvokableClass(AbstractInvokable.class);
	jobVertex.setParallelism(parallelism);

	if (NOT_CONFIGURED != preconfiguredMaxParallelism) {
		jobVertex.setMaxParallelism(preconfiguredMaxParallelism);
	}

	ExecutionGraph executionGraphMock = mock(ExecutionGraph.class);
	when(executionGraphMock.getFutureExecutor()).thenReturn(Executors.directExecutor());
	ExecutionJobVertex executionJobVertex =
			new ExecutionJobVertex(executionGraphMock, jobVertex, 1, Time.seconds(10));

	return executionJobVertex;
}
 
Example #5
Source File: WebMonitorUtilsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests dynamically loading of handlers such as {@link JarUploadHandler}.
 */
@Test
public void testLoadWebSubmissionExtension() throws Exception {
	final Configuration configuration = new Configuration();
	configuration.setString(JobManagerOptions.ADDRESS, "localhost");
	final WebMonitorExtension webMonitorExtension = WebMonitorUtils.loadWebSubmissionExtension(
		CompletableFuture::new,
		Time.seconds(10),
		Collections.emptyMap(),
		CompletableFuture.completedFuture("localhost:12345"),
		Paths.get("/tmp"),
		Executors.directExecutor(),
		configuration);

	assertThat(webMonitorExtension, is(not(nullValue())));
}
 
Example #6
Source File: CheckpointCoordinatorMasterHooksTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private static CheckpointCoordinator instantiateCheckpointCoordinator(JobID jid, ExecutionVertex... ackVertices) {
	return new CheckpointCoordinator(
			jid,
			10000000L,
			600000L,
			0L,
			1,
			CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
			new ExecutionVertex[0],
			ackVertices,
			new ExecutionVertex[0],
			new StandaloneCheckpointIDCounter(),
			new StandaloneCompletedCheckpointStore(10),
			new MemoryStateBackend(),
			Executors.directExecutor(),
			SharedStateRegistry.DEFAULT_FACTORY);
}
 
Example #7
Source File: SlotManagerFailUnfulfillableTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private static SlotManager createSlotManager(
		List<Tuple3<JobID, AllocationID, Exception>> notifiedAllocationFailures,
		boolean startNewTMs) {

	final ResourceActions resourceManagerActions = new TestingResourceActionsBuilder()
		.setAllocateResourceFunction((resourceProfile) -> startNewTMs ?
						Collections.singleton(resourceProfile) :
						Collections.emptyList())
		.setNotifyAllocationFailureConsumer(tuple3 -> notifiedAllocationFailures.add(tuple3))
		.build();

	SlotManager slotManager = SlotManagerBuilder.newBuilder().build();
	slotManager.start(ResourceManagerId.generate(), Executors.directExecutor(), resourceManagerActions);

	return slotManager;
}
 
Example #8
Source File: ExecutionJobVertexTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private static ExecutionJobVertex createExecutionJobVertex(
		int parallelism,
		int preconfiguredMaxParallelism) throws JobException {

	JobVertex jobVertex = new JobVertex("testVertex");
	jobVertex.setInvokableClass(AbstractInvokable.class);
	jobVertex.setParallelism(parallelism);

	if (NOT_CONFIGURED != preconfiguredMaxParallelism) {
		jobVertex.setMaxParallelism(preconfiguredMaxParallelism);
	}

	ExecutionGraph executionGraphMock = mock(ExecutionGraph.class);
	when(executionGraphMock.getFutureExecutor()).thenReturn(Executors.directExecutor());
	ExecutionJobVertex executionJobVertex =
			new ExecutionJobVertex(executionGraphMock, jobVertex, 1, Time.seconds(10));

	return executionJobVertex;
}
 
Example #9
Source File: CheckpointCoordinatorMasterHooksTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private static CheckpointCoordinator instantiateCheckpointCoordinator(JobID jid, ExecutionVertex... ackVertices) {
	CheckpointCoordinatorConfiguration chkConfig = new CheckpointCoordinatorConfiguration(
		10000000L,
		600000L,
		0L,
		1,
		CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
		true,
		false,
		0);
	return new CheckpointCoordinator(
			jid,
			chkConfig,
			new ExecutionVertex[0],
			ackVertices,
			new ExecutionVertex[0],
			new StandaloneCheckpointIDCounter(),
			new StandaloneCompletedCheckpointStore(10),
			new MemoryStateBackend(),
			Executors.directExecutor(),
			SharedStateRegistry.DEFAULT_FACTORY,
			new CheckpointFailureManager(
				0,
				NoOpFailJobCall.INSTANCE));
}
 
Example #10
Source File: ZooKeeperCompletedCheckpointStoreITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
protected ZooKeeperCompletedCheckpointStore createCompletedCheckpoints(int maxNumberOfCheckpointsToRetain) throws Exception {
	final ZooKeeperStateHandleStore<CompletedCheckpoint> checkpointsInZooKeeper = ZooKeeperUtils.createZooKeeperStateHandleStore(
		ZOOKEEPER.getClient(),
		CHECKPOINT_PATH,
		new TestingRetrievableStateStorageHelper<>());

	return new ZooKeeperCompletedCheckpointStore(
		maxNumberOfCheckpointsToRetain,
		checkpointsInZooKeeper,
		Executors.directExecutor());
}
 
Example #11
Source File: JarDeleteHandlerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
	jarDir = temporaryFolder.newFolder().toPath();
	restfulGateway = TestingRestfulGateway.newBuilder().build();
	jarDeleteHandler = new JarDeleteHandler(
		() -> CompletableFuture.completedFuture(restfulGateway),
		Time.seconds(10),
		Collections.emptyMap(),
		new JarDeleteHeaders(),
		jarDir,
		Executors.directExecutor()
	);

	Files.createFile(jarDir.resolve(TEST_JAR_NAME));
}
 
Example #12
Source File: ZooKeeperRegistryTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the function of ZookeeperRegistry, setJobRunning(), setJobFinished(), isJobRunning()
 */
@Test
public void testZooKeeperRegistry() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

	final HighAvailabilityServices zkHaService = new ZooKeeperHaServices(
			ZooKeeperUtils.startCuratorFramework(configuration),
		Executors.directExecutor(),
		configuration,
		new VoidBlobStore());

	final RunningJobsRegistry zkRegistry = zkHaService.getRunningJobsRegistry();

	try {
		JobID jobID = JobID.generate();
		assertEquals(JobSchedulingStatus.PENDING, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.setJobRunning(jobID);
		assertEquals(JobSchedulingStatus.RUNNING, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.setJobFinished(jobID);
		assertEquals(JobSchedulingStatus.DONE, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.clearJob(jobID);
		assertEquals(JobSchedulingStatus.PENDING, zkRegistry.getJobSchedulingStatus(jobID));
	} finally {
		zkHaService.close();
	}
}
 
Example #13
Source File: JarUploadHandlerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
	MockitoAnnotations.initMocks(this);

	jarDir = temporaryFolder.newFolder().toPath();
	jarUploadHandler = new JarUploadHandler(
		() -> CompletableFuture.completedFuture(mockDispatcherGateway),
		Time.seconds(10),
		Collections.emptyMap(),
		JarUploadHeaders.getInstance(),
		jarDir,
		Executors.directExecutor());
}
 
Example #14
Source File: SlotManagerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that a task manager timeout does not remove the slots from the SlotManager.
 * A timeout should only trigger the {@link ResourceActions#releaseResource(InstanceID, Exception)}
 * callback. The receiver of the callback can then decide what to do with the TaskManager.
 *
 * <p>See FLINK-7793
 */
@Test
public void testTaskManagerTimeoutDoesNotRemoveSlots() throws Exception {
	final Time taskManagerTimeout = Time.milliseconds(10L);
	final ResourceManagerId resourceManagerId = ResourceManagerId.generate();
	final ResourceID resourceID = ResourceID.generate();
	final ResourceActions resourceActions = mock(ResourceActions.class);
	final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);

	when(taskExecutorGateway.canBeReleased()).thenReturn(CompletableFuture.completedFuture(true));

	final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(resourceID, taskExecutorGateway);
	final SlotStatus slotStatus = new SlotStatus(
		new SlotID(resourceID, 0),
		new ResourceProfile(1.0, 1));
	final SlotReport initialSlotReport = new SlotReport(slotStatus);

	try (final SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setTaskManagerTimeout(taskManagerTimeout)
		.build()) {

		slotManager.start(resourceManagerId, Executors.directExecutor(), resourceActions);

		slotManager.registerTaskManager(taskExecutorConnection, initialSlotReport);

		assertEquals(1, slotManager.getNumberRegisteredSlots());

		// wait for the timeout call to happen
		verify(resourceActions, timeout(taskManagerTimeout.toMilliseconds() * 20L).atLeast(1)).releaseResource(eq(taskExecutorConnection.getInstanceID()), any(Exception.class));

		assertEquals(1, slotManager.getNumberRegisteredSlots());

		slotManager.unregisterTaskManager(taskExecutorConnection.getInstanceID());

		assertEquals(0, slotManager.getNumberRegisteredSlots());
	}
}
 
Example #15
Source File: TaskLocalStateStoreImplTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Before
public void before() throws Exception {
	JobID jobID = new JobID();
	AllocationID allocationID = new AllocationID();
	JobVertexID jobVertexID = new JobVertexID();
	int subtaskIdx = 0;
	this.temporaryFolder = new TemporaryFolder();
	this.temporaryFolder.create();
	this.allocationBaseDirs = new File[]{temporaryFolder.newFolder(), temporaryFolder.newFolder()};
	this.internalSnapshotMap = new TreeMap<>();
	this.internalLock = new Object();

	LocalRecoveryDirectoryProviderImpl directoryProvider =
		new LocalRecoveryDirectoryProviderImpl(allocationBaseDirs, jobID, jobVertexID, subtaskIdx);

	LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig(false, directoryProvider);

	this.taskLocalStateStore = new TaskLocalStateStoreImpl(
		jobID,
		allocationID,
		jobVertexID,
		subtaskIdx,
		localRecoveryConfig,
		Executors.directExecutor(),
		internalSnapshotMap,
		internalLock);
}
 
Example #16
Source File: TaskExecutorLocalStateStoresManagerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * This tests that the creation of {@link TaskManagerServices} correctly falls back to the first tmp directory of
 * the IOManager as default for the local state root directory.
 */
@Test
public void testCreationFromConfigDefault() throws Exception {

	final Configuration config = new Configuration();

	final ResourceID tmResourceID = ResourceID.generate();

	TaskManagerServicesConfiguration taskManagerServicesConfiguration =
		TaskManagerServicesConfiguration.fromConfiguration(config, InetAddress.getLocalHost(), true);

	TaskManagerServices taskManagerServices = TaskManagerServices.fromConfiguration(
		taskManagerServicesConfiguration,
		tmResourceID,
		Executors.directExecutor(),
		MEM_SIZE_PARAM,
		MEM_SIZE_PARAM);

	TaskExecutorLocalStateStoresManager taskStateManager = taskManagerServices.getTaskManagerStateStore();

	String[] tmpDirPaths = taskManagerServicesConfiguration.getTmpDirPaths();
	File[] localStateRootDirectories = taskStateManager.getLocalStateRootDirectories();

	for (int i = 0; i < tmpDirPaths.length; ++i) {
		Assert.assertEquals(
			new File(tmpDirPaths[i], TaskManagerServices.LOCAL_STATE_SUB_DIRECTORY_ROOT),
			localStateRootDirectories[i]);
	}

	Assert.assertFalse(taskStateManager.isLocalRecoveryEnabled());
}
 
Example #17
Source File: TaskExecutorToResourceManagerConnectionTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private TaskExecutorToResourceManagerConnection createTaskExecutorToResourceManagerConnection() {
	return new TaskExecutorToResourceManagerConnection(
		LOGGER,
		rpcService,
		TASK_MANAGER_ADDRESS,
		TASK_MANAGER_RESOURCE_ID,
		RetryingRegistrationConfiguration.defaultConfiguration(),
		TASK_MANAGER_DATA_PORT,
		TASK_MANAGER_HARDWARE_DESCRIPTION,
		RESOURCE_MANAGER_ADDRESS,
		RESOURCE_MANAGER_ID,
		Executors.directExecutor(),
		new TestRegistrationConnectionListener<>());
}
 
Example #18
Source File: SlotManagerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that a task manager timeout does not remove the slots from the SlotManager.
 * A timeout should only trigger the {@link ResourceActions#releaseResource(InstanceID, Exception)}
 * callback. The receiver of the callback can then decide what to do with the TaskManager.
 *
 * <p>See FLINK-7793
 */
@Test
public void testTaskManagerTimeoutDoesNotRemoveSlots() throws Exception {
	final Time taskManagerTimeout = Time.milliseconds(10L);
	final ResourceManagerId resourceManagerId = ResourceManagerId.generate();
	final ResourceID resourceID = ResourceID.generate();
	final ResourceActions resourceActions = mock(ResourceActions.class);
	final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);

	when(taskExecutorGateway.canBeReleased()).thenReturn(CompletableFuture.completedFuture(true));

	final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(resourceID, taskExecutorGateway);
	final SlotStatus slotStatus = new SlotStatus(
		new SlotID(resourceID, 0),
		new ResourceProfile(1.0, 1));
	final SlotReport initialSlotReport = new SlotReport(slotStatus);

	try (final SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setTaskManagerTimeout(taskManagerTimeout)
		.build()) {

		slotManager.start(resourceManagerId, Executors.directExecutor(), resourceActions);

		slotManager.registerTaskManager(taskExecutorConnection, initialSlotReport);

		assertEquals(1, slotManager.getNumberRegisteredSlots());

		// wait for the timeout call to happen
		verify(resourceActions, timeout(taskManagerTimeout.toMilliseconds() * 20L).atLeast(1)).releaseResource(eq(taskExecutorConnection.getInstanceID()), any(Exception.class));

		assertEquals(1, slotManager.getNumberRegisteredSlots());

		slotManager.unregisterTaskManager(taskExecutorConnection.getInstanceID());

		assertEquals(0, slotManager.getNumberRegisteredSlots());
	}
}
 
Example #19
Source File: MetricFetcherTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Nonnull
private MetricFetcher createMetricFetcher(long updateInterval, RestfulGateway restfulGateway) {
	return new MetricFetcherImpl<>(
		() -> CompletableFuture.completedFuture(restfulGateway),
		path -> new CompletableFuture<>(),
		Executors.directExecutor(),
		Time.seconds(10L),
		updateInterval);
}
 
Example #20
Source File: ZooKeeperRegistryTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the function of ZookeeperRegistry, setJobRunning(), setJobFinished(), isJobRunning()
 */
@Test
public void testZooKeeperRegistry() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

	final HighAvailabilityServices zkHaService = new ZooKeeperHaServices(
			ZooKeeperUtils.startCuratorFramework(configuration),
		Executors.directExecutor(),
		configuration,
		new VoidBlobStore());

	final RunningJobsRegistry zkRegistry = zkHaService.getRunningJobsRegistry();

	try {
		JobID jobID = JobID.generate();
		assertEquals(JobSchedulingStatus.PENDING, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.setJobRunning(jobID);
		assertEquals(JobSchedulingStatus.RUNNING, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.setJobFinished(jobID);
		assertEquals(JobSchedulingStatus.DONE, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.clearJob(jobID);
		assertEquals(JobSchedulingStatus.PENDING, zkRegistry.getJobSchedulingStatus(jobID));
	} finally {
		zkHaService.close();
	}
}
 
Example #21
Source File: ZooKeeperHaServicesTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private void runCleanupTest(
		Configuration configuration,
		TestingBlobStoreService blobStoreService,
		ThrowingConsumer<ZooKeeperHaServices, Exception> zooKeeperHaServicesConsumer) throws Exception {
	try (ZooKeeperHaServices zooKeeperHaServices = new ZooKeeperHaServices(
		ZooKeeperUtils.startCuratorFramework(configuration),
		Executors.directExecutor(),
		configuration,
		blobStoreService)) {

		// create some Zk services to trigger the generation of paths
		final LeaderRetrievalService resourceManagerLeaderRetriever = zooKeeperHaServices.getResourceManagerLeaderRetriever();
		final LeaderElectionService resourceManagerLeaderElectionService = zooKeeperHaServices.getResourceManagerLeaderElectionService();
		final RunningJobsRegistry runningJobsRegistry = zooKeeperHaServices.getRunningJobsRegistry();

		final TestingListener listener = new TestingListener();
		resourceManagerLeaderRetriever.start(listener);
		resourceManagerLeaderElectionService.start(new TestingContender("foobar", resourceManagerLeaderElectionService));
		final JobID jobId = new JobID();
		runningJobsRegistry.setJobRunning(jobId);

		listener.waitForNewLeader(2000L);

		resourceManagerLeaderRetriever.stop();
		resourceManagerLeaderElectionService.stop();
		runningJobsRegistry.clearJob(jobId);

		zooKeeperHaServicesConsumer.accept(zooKeeperHaServices);
	}
}
 
Example #22
Source File: ClusterClient.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a instance that submits the programs to the JobManager defined in the
 * configuration. This method will try to resolve the JobManager hostname and throw an exception
 * if that is not possible.
 *
 * @param flinkConfig The config used to obtain the job-manager's address, and used to configure the optimizer.
 *
 * @throws Exception we cannot create the high availability services
 */
public ClusterClient(Configuration flinkConfig) throws Exception {
	this(
		flinkConfig,
		HighAvailabilityServicesUtils.createHighAvailabilityServices(
			flinkConfig,
			Executors.directExecutor(),
			HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION),
		false);
}
 
Example #23
Source File: JarDeleteHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
	jarDir = temporaryFolder.newFolder().toPath();
	restfulGateway = TestingRestfulGateway.newBuilder().build();
	jarDeleteHandler = new JarDeleteHandler(
		() -> CompletableFuture.completedFuture(restfulGateway),
		Time.seconds(10),
		Collections.emptyMap(),
		new JarDeleteHeaders(),
		jarDir,
		Executors.directExecutor()
	);

	Files.createFile(jarDir.resolve(TEST_JAR_NAME));
}
 
Example #24
Source File: ZooKeeperCompletedCheckpointStoreTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Nonnull
private ZooKeeperCompletedCheckpointStore createZooKeeperCheckpointStore(CuratorFramework client) throws Exception {
	final ZooKeeperStateHandleStore<CompletedCheckpoint> checkpointsInZooKeeper = ZooKeeperUtils.createZooKeeperStateHandleStore(
		client,
		"/checkpoints",
		new TestingRetrievableStateStorageHelper<>());

	return new ZooKeeperCompletedCheckpointStore(
		1,
		checkpointsInZooKeeper,
		Executors.directExecutor());
}
 
Example #25
Source File: ZooKeeperCompletedCheckpointStoreITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected ZooKeeperCompletedCheckpointStore createCompletedCheckpoints(int maxNumberOfCheckpointsToRetain) throws Exception {
	final ZooKeeperStateHandleStore<CompletedCheckpoint> checkpointsInZooKeeper = ZooKeeperUtils.createZooKeeperStateHandleStore(
		ZOOKEEPER.getClient(),
		CHECKPOINT_PATH,
		new TestingRetrievableStateStorageHelper<>());

	return new ZooKeeperCompletedCheckpointStore(
		maxNumberOfCheckpointsToRetain,
		checkpointsInZooKeeper,
		Executors.directExecutor());
}
 
Example #26
Source File: TaskExecutorLocalStateStoresManagerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private TaskManagerServices createTaskManagerServices(
		TaskManagerServicesConfiguration config) throws Exception {
	return TaskManagerServices.fromConfiguration(
		config,
		UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(),
		Executors.directExecutor());
}
 
Example #27
Source File: TaskLocalStateStoreImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Before
public void before() throws Exception {
	JobID jobID = new JobID();
	AllocationID allocationID = new AllocationID();
	JobVertexID jobVertexID = new JobVertexID();
	int subtaskIdx = 0;
	this.temporaryFolder = new TemporaryFolder();
	this.temporaryFolder.create();
	this.allocationBaseDirs = new File[]{temporaryFolder.newFolder(), temporaryFolder.newFolder()};
	this.internalSnapshotMap = new TreeMap<>();
	this.internalLock = new Object();

	LocalRecoveryDirectoryProviderImpl directoryProvider =
		new LocalRecoveryDirectoryProviderImpl(allocationBaseDirs, jobID, jobVertexID, subtaskIdx);

	LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig(false, directoryProvider);

	this.taskLocalStateStore = new TaskLocalStateStoreImpl(
		jobID,
		allocationID,
		jobVertexID,
		subtaskIdx,
		localRecoveryConfig,
		Executors.directExecutor(),
		internalSnapshotMap,
		internalLock);
}
 
Example #28
Source File: CheckpointCoordinatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private CheckpointCoordinator getCheckpointCoordinator(
		final JobID jobId,
		final ExecutionVertex vertex1,
		final ExecutionVertex vertex2,
		final CheckpointFailureManager failureManager) {

	final CheckpointCoordinatorConfiguration chkConfig = new CheckpointCoordinatorConfiguration(
			600000,
			600000,
			0,
			Integer.MAX_VALUE,
			CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
			true,
			false,
			0);

	return new CheckpointCoordinator(
			jobId,
			chkConfig,
			new ExecutionVertex[]{vertex1, vertex2},
			new ExecutionVertex[]{vertex1, vertex2},
			new ExecutionVertex[]{vertex1, vertex2},
			new StandaloneCheckpointIDCounter(),
			new StandaloneCompletedCheckpointStore(1),
			new MemoryStateBackend(),
			Executors.directExecutor(),
			SharedStateRegistry.DEFAULT_FACTORY,
			failureManager);
}
 
Example #29
Source File: MetricFetcherTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Nonnull
private MetricFetcher createMetricFetcher(long updateInterval, RestfulGateway restfulGateway) {
	return new MetricFetcherImpl<>(
		() -> CompletableFuture.completedFuture(restfulGateway),
		address -> null,
		Executors.directExecutor(),
		Time.seconds(10L),
		updateInterval);
}
 
Example #30
Source File: TaskExecutorToResourceManagerConnectionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private TaskExecutorToResourceManagerConnection createTaskExecutorToResourceManagerConnection() {
	return new TaskExecutorToResourceManagerConnection(
		LOGGER,
		rpcService,
		TASK_MANAGER_ADDRESS,
		TASK_MANAGER_RESOURCE_ID,
		RetryingRegistrationConfiguration.defaultConfiguration(),
		TASK_MANAGER_DATA_PORT,
		TASK_MANAGER_HARDWARE_DESCRIPTION,
		RESOURCE_MANAGER_ADDRESS,
		RESOURCE_MANAGER_ID,
		Executors.directExecutor(),
		new TestRegistrationConnectionListener<>());
}