org.apache.flink.runtime.util.ZooKeeperUtils Java Examples

The following examples show how to use org.apache.flink.runtime.util.ZooKeeperUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ZooKeeperMesosServices.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public MesosWorkerStore createMesosWorkerStore(Configuration configuration, Executor executor) throws Exception {
	RetrievableStateStorageHelper<MesosWorkerStore.Worker> stateStorageHelper =
		ZooKeeperUtils.createFileSystemStateStorage(configuration, "mesosWorkerStore");

	ZooKeeperStateHandleStore<MesosWorkerStore.Worker> zooKeeperStateHandleStore = zooKeeperUtilityFactory.createZooKeeperStateHandleStore(
		"/workers",
		stateStorageHelper);

	ZooKeeperSharedValue frameworkId = zooKeeperUtilityFactory.createSharedValue("/frameworkId", new byte[0]);
	ZooKeeperSharedCount totalTaskCount = zooKeeperUtilityFactory.createSharedCount("/taskCount", 0);

	return new ZooKeeperMesosWorkerStore(
		zooKeeperStateHandleStore,
		frameworkId,
		totalTaskCount);
}
 
Example #2
Source File: ZooKeeperCompletedCheckpointStoreTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that checkpoints are discarded when the completed checkpoint store is shut
 * down with a globally terminal state.
 */
@Test
public void testDiscardingCheckpointsAtShutDown() throws Exception {
	final SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	final Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());

	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final ZooKeeperCompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(client);

	try {
		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);

		checkpointStore.addCheckpoint(checkpoint1);
		assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));

		checkpointStore.shutdown(JobStatus.FINISHED);

		// verify that the checkpoint is discarded
		CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
	} finally {
		client.close();
	}
}
 
Example #3
Source File: HighAvailabilityServicesUtils.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public static HighAvailabilityServices createAvailableOrEmbeddedServices(
	Configuration config,
	Executor executor) throws Exception {
	HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(config);

	switch (highAvailabilityMode) {
		case NONE:
			return new EmbeddedHaServices(executor);

		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(config),
				executor,
				config,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(config, executor);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 
Example #4
Source File: ZooKeeperLeaderRetrievalTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Before
public void before() throws Exception {
	testingServer = new TestingServer();

	config = new Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
	config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());

	CuratorFramework client = ZooKeeperUtils.startCuratorFramework(config);

	highAvailabilityServices = new ZooKeeperHaServices(
		client,
		TestingUtils.defaultExecutor(),
		config,
		new VoidBlobStore());
}
 
Example #5
Source File: ZooKeeperCompletedCheckpointStoreTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that checkpoints are discarded when the completed checkpoint store is shut
 * down with a globally terminal state.
 */
@Test
public void testDiscardingCheckpointsAtShutDown() throws Exception {
	final SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	final Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());

	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final ZooKeeperCompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(client);

	try {
		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);

		checkpointStore.addCheckpoint(checkpoint1);
		assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));

		checkpointStore.shutdown(JobStatus.FINISHED);

		// verify that the checkpoint is discarded
		CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
	} finally {
		client.close();
	}
}
 
Example #6
Source File: ZooKeeperLeaderRetrievalTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Before
public void before() throws Exception {
	testingServer = new TestingServer();

	config = new Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
	config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());

	CuratorFramework client = ZooKeeperUtils.startCuratorFramework(config);

	highAvailabilityServices = new ZooKeeperHaServices(
		client,
		TestingUtils.defaultExecutor(),
		config,
		new VoidBlobStore());
}
 
Example #7
Source File: HighAvailabilityServicesUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
public static HighAvailabilityServices createAvailableOrEmbeddedServices(
	Configuration config,
	Executor executor) throws Exception {
	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(config);

	switch (highAvailabilityMode) {
		case NONE:
			return new EmbeddedHaServices(executor);

		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(config),
				executor,
				config,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(config, executor);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 
Example #8
Source File: ZooKeeperMesosServices.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public MesosWorkerStore createMesosWorkerStore(Configuration configuration, Executor executor) throws Exception {
	RetrievableStateStorageHelper<MesosWorkerStore.Worker> stateStorageHelper =
		ZooKeeperUtils.createFileSystemStateStorage(configuration, "mesosWorkerStore");

	ZooKeeperStateHandleStore<MesosWorkerStore.Worker> zooKeeperStateHandleStore = zooKeeperUtilityFactory.createZooKeeperStateHandleStore(
		"/workers",
		stateStorageHelper);

	ZooKeeperSharedValue frameworkId = zooKeeperUtilityFactory.createSharedValue("/frameworkId", new byte[0]);
	ZooKeeperSharedCount totalTaskCount = zooKeeperUtilityFactory.createSharedCount("/taskCount", 0);

	return new ZooKeeperMesosWorkerStore(
		zooKeeperStateHandleStore,
		frameworkId,
		totalTaskCount);
}
 
Example #9
Source File: ZooKeeperCompletedCheckpointStoreTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that checkpoints are discarded when the completed checkpoint store is shut
 * down with a globally terminal state.
 */
@Test
public void testDiscardingCheckpointsAtShutDown() throws Exception {
	final SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	final Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());

	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final ZooKeeperCompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(client);

	try {
		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);

		checkpointStore.addCheckpoint(checkpoint1);
		assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));

		checkpointStore.shutdown(JobStatus.FINISHED);

		// verify that the checkpoint is discarded
		CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
	} finally {
		client.close();
	}
}
 
Example #10
Source File: HighAvailabilityServicesUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
public static HighAvailabilityServices createAvailableOrEmbeddedServices(
	Configuration config,
	Executor executor) throws Exception {
	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(config);

	switch (highAvailabilityMode) {
		case NONE:
			return new EmbeddedHaServices(executor);

		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(config),
				executor,
				config,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(config, executor);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 
Example #11
Source File: ZooKeeperRegistryTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the function of ZookeeperRegistry, setJobRunning(), setJobFinished(), isJobRunning()
 */
@Test
public void testZooKeeperRegistry() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

	final HighAvailabilityServices zkHaService = new ZooKeeperHaServices(
			ZooKeeperUtils.startCuratorFramework(configuration),
		Executors.directExecutor(),
		configuration,
		new VoidBlobStore());

	final RunningJobsRegistry zkRegistry = zkHaService.getRunningJobsRegistry();

	try {
		JobID jobID = JobID.generate();
		assertEquals(JobSchedulingStatus.PENDING, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.setJobRunning(jobID);
		assertEquals(JobSchedulingStatus.RUNNING, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.setJobFinished(jobID);
		assertEquals(JobSchedulingStatus.DONE, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.clearJob(jobID);
		assertEquals(JobSchedulingStatus.PENDING, zkRegistry.getJobSchedulingStatus(jobID));
	} finally {
		zkHaService.close();
	}
}
 
Example #12
Source File: ZooKeeperHaServicesTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void runCleanupTest(
		Configuration configuration,
		TestingBlobStoreService blobStoreService,
		ThrowingConsumer<ZooKeeperHaServices, Exception> zooKeeperHaServicesConsumer) throws Exception {
	try (ZooKeeperHaServices zooKeeperHaServices = new ZooKeeperHaServices(
		ZooKeeperUtils.startCuratorFramework(configuration),
		Executors.directExecutor(),
		configuration,
		blobStoreService)) {

		// create some Zk services to trigger the generation of paths
		final LeaderRetrievalService resourceManagerLeaderRetriever = zooKeeperHaServices.getResourceManagerLeaderRetriever();
		final LeaderElectionService resourceManagerLeaderElectionService = zooKeeperHaServices.getResourceManagerLeaderElectionService();
		final RunningJobsRegistry runningJobsRegistry = zooKeeperHaServices.getRunningJobsRegistry();

		final TestingListener listener = new TestingListener();
		resourceManagerLeaderRetriever.start(listener);
		resourceManagerLeaderElectionService.start(new TestingContender("foobar", resourceManagerLeaderElectionService));
		final JobID jobId = new JobID();
		runningJobsRegistry.setJobRunning(jobId);

		listener.waitForNewLeader(2000L);

		resourceManagerLeaderRetriever.stop();
		resourceManagerLeaderElectionService.stop();
		runningJobsRegistry.clearJob(jobId);

		zooKeeperHaServicesConsumer.accept(zooKeeperHaServices);
	}
}
 
Example #13
Source File: ZooKeeperStateHandleStoreTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testDeleteAllShouldRemoveAllPaths() throws Exception {
	final ZooKeeperStateHandleStore<Long> zkStore = new ZooKeeperStateHandleStore<>(
		ZooKeeperUtils.useNamespaceAndEnsurePath(ZOOKEEPER.getClient(), "/path"),
		new LongStateStorage());

	zkStore.addAndLock("/state", 1L);
	zkStore.deleteChildren();

	assertThat(zkStore.getAllPaths(), is(empty()));
}
 
Example #14
Source File: ZooKeeperLeaderElectionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the ZooKeeperLeaderElection/RetrievalService return both the correct URL.
 */
@Test
public void testZooKeeperLeaderElectionRetrieval() throws Exception {
	ZooKeeperLeaderElectionService leaderElectionService = null;
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	try {
		leaderElectionService = ZooKeeperUtils.createLeaderElectionService(client, configuration);
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		TestingContender contender = new TestingContender(TEST_URL, leaderElectionService);
		TestingListener listener = new TestingListener();

		leaderElectionService.start(contender);
		leaderRetrievalService.start(listener);

		contender.waitForLeader(timeout);

		assertTrue(contender.isLeader());
		assertEquals(leaderElectionService.getLeaderSessionID(), contender.getLeaderSessionID());

		listener.waitForNewLeader(timeout);

		assertEquals(TEST_URL, listener.getAddress());
		assertEquals(leaderElectionService.getLeaderSessionID(), listener.getLeaderSessionID());

	} finally {
		if (leaderElectionService != null) {
			leaderElectionService.stop();
		}

		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}
	}
}
 
Example #15
Source File: ZooKeeperLeaderElectionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Before
public void before() {
	try {
		testingServer = new TestingServer();
	} catch (Exception e) {
		throw new RuntimeException("Could not start ZooKeeper testing cluster.", e);
	}

	configuration = new Configuration();

	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

	client = ZooKeeperUtils.startCuratorFramework(configuration);
}
 
Example #16
Source File: ZooKeeperSubmittedJobGraphStoreTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that we fail with an exception if the job cannot be removed from the
 * ZooKeeperSubmittedJobGraphStore.
 *
 * <p>Tests that a close ZooKeeperSubmittedJobGraphStore no longer holds any locks.
 */
@Test
public void testJobGraphRemovalFailureAndLockRelease() throws Exception {
	try (final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration)) {
		final TestingRetrievableStateStorageHelper<SubmittedJobGraph> stateStorage = new TestingRetrievableStateStorageHelper<>();
		final ZooKeeperSubmittedJobGraphStore submittedJobGraphStore = createSubmittedJobGraphStore(client, stateStorage);
		submittedJobGraphStore.start(null);
		final ZooKeeperSubmittedJobGraphStore otherSubmittedJobGraphStore = createSubmittedJobGraphStore(client, stateStorage);
		otherSubmittedJobGraphStore.start(null);

		final SubmittedJobGraph jobGraph = new SubmittedJobGraph(new JobGraph());
		submittedJobGraphStore.putJobGraph(jobGraph);

		final SubmittedJobGraph recoveredJobGraph = otherSubmittedJobGraphStore.recoverJobGraph(jobGraph.getJobId());

		assertThat(recoveredJobGraph, is(notNullValue()));

		try {
			otherSubmittedJobGraphStore.removeJobGraph(recoveredJobGraph.getJobId());
			fail("It should not be possible to remove the JobGraph since the first store still has a lock on it.");
		} catch (Exception ignored) {
			// expected
		}

		submittedJobGraphStore.stop();

		// now we should be able to delete the job graph
		otherSubmittedJobGraphStore.removeJobGraph(recoveredJobGraph.getJobId());

		assertThat(otherSubmittedJobGraphStore.recoverJobGraph(recoveredJobGraph.getJobId()), is(nullValue()));

		otherSubmittedJobGraphStore.stop();
	}
}
 
Example #17
Source File: ZooKeeperLeaderElectionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the ZooKeeperLeaderElection/RetrievalService return both the correct URL.
 */
@Test
public void testZooKeeperLeaderElectionRetrieval() throws Exception {
	ZooKeeperLeaderElectionService leaderElectionService = null;
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	try {
		leaderElectionService = ZooKeeperUtils.createLeaderElectionService(client, configuration);
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		TestingContender contender = new TestingContender(TEST_URL, leaderElectionService);
		TestingListener listener = new TestingListener();

		leaderElectionService.start(contender);
		leaderRetrievalService.start(listener);

		contender.waitForLeader(timeout);

		assertTrue(contender.isLeader());
		assertEquals(leaderElectionService.getLeaderSessionID(), contender.getLeaderSessionID());

		listener.waitForNewLeader(timeout);

		assertEquals(TEST_URL, listener.getAddress());
		assertEquals(leaderElectionService.getLeaderSessionID(), listener.getLeaderSessionID());

	} finally {
		if (leaderElectionService != null) {
			leaderElectionService.stop();
		}

		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}
	}
}
 
Example #18
Source File: LeaderElectionTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void setup() throws Exception {
	try {
		testingServer = new TestingServer();
	} catch (Exception e) {
		throw new RuntimeException("Could not start ZooKeeper testing cluster.", e);
	}

	configuration = new Configuration();

	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

	client = ZooKeeperUtils.startCuratorFramework(configuration);
}
 
Example #19
Source File: ZooKeeperLeaderElectionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Before
public void before() {
	try {
		testingServer = new TestingServer();
	} catch (Exception e) {
		throw new RuntimeException("Could not start ZooKeeper testing cluster.", e);
	}

	configuration = new Configuration();

	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

	client = ZooKeeperUtils.startCuratorFramework(configuration);
}
 
Example #20
Source File: ZooKeeperStateHandleStoreTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * FLINK-6612
 *
 * Tests that lock nodes will be released if the client dies.
 */
@Test
public void testLockCleanupWhenClientTimesOut() throws Exception {
	LongStateStorage longStateStorage = new LongStateStorage();

	Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, ZOOKEEPER.getConnectString());
	configuration.setInteger(HighAvailabilityOptions.ZOOKEEPER_SESSION_TIMEOUT, 100);
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_ROOT, "timeout");

	try (CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
		CuratorFramework client2 = ZooKeeperUtils.startCuratorFramework(configuration)) {

		ZooKeeperStateHandleStore<Long> zkStore = new ZooKeeperStateHandleStore<>(
			client,
			longStateStorage);

		final String path = "/state";

		zkStore.addAndLock(path, 42L);

		// this should delete all ephemeral nodes
		client.close();

		Stat stat = client2.checkExists().forPath(path);

		// check that our state node still exists
		assertNotNull(stat);

		Collection<String> children = client2.getChildren().forPath(path);

		// check that the lock node has been released
		assertEquals(0, children.size());
	}
}
 
Example #21
Source File: ZooKeeperStateHandleStoreTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testDeleteAllShouldRemoveAllPaths() throws Exception {
	final ZooKeeperStateHandleStore<Long> zkStore = new ZooKeeperStateHandleStore<>(
		ZooKeeperUtils.useNamespaceAndEnsurePath(ZOOKEEPER.getClient(), "/path"),
		new LongStateStorage());

	zkStore.addAndLock("/state", 1L);
	zkStore.deleteChildren();

	assertThat(zkStore.getAllPaths(), is(empty()));
}
 
Example #22
Source File: ZooKeeperCompletedCheckpointStoreTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that subsumed checkpoints are discarded.
 */
@Test
public void testDiscardingSubsumedCheckpoints() throws Exception {
	final SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	final Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());

	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final ZooKeeperCompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(client);

	try {
		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);

		checkpointStore.addCheckpoint(checkpoint1);
		assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));

		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint2 = CompletedCheckpointStoreTest.createCheckpoint(1, sharedStateRegistry);
		checkpointStore.addCheckpoint(checkpoint2);
		final List<CompletedCheckpoint> allCheckpoints = checkpointStore.getAllCheckpoints();
		assertThat(allCheckpoints, Matchers.contains(checkpoint2));
		assertThat(allCheckpoints, Matchers.not(Matchers.contains(checkpoint1)));

		// verify that the subsumed checkpoint is discarded
		CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
	} finally {
		client.close();
	}
}
 
Example #23
Source File: ZooKeeperSubmittedJobGraphStoreTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that we fail with an exception if the job cannot be removed from the
 * ZooKeeperSubmittedJobGraphStore.
 *
 * <p>Tests that a close ZooKeeperSubmittedJobGraphStore no longer holds any locks.
 */
@Test
public void testJobGraphRemovalFailureAndLockRelease() throws Exception {
	try (final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration)) {
		final TestingRetrievableStateStorageHelper<SubmittedJobGraph> stateStorage = new TestingRetrievableStateStorageHelper<>();
		final ZooKeeperSubmittedJobGraphStore submittedJobGraphStore = createSubmittedJobGraphStore(client, stateStorage);
		submittedJobGraphStore.start(null);
		final ZooKeeperSubmittedJobGraphStore otherSubmittedJobGraphStore = createSubmittedJobGraphStore(client, stateStorage);
		otherSubmittedJobGraphStore.start(null);

		final SubmittedJobGraph jobGraph = new SubmittedJobGraph(new JobGraph());
		submittedJobGraphStore.putJobGraph(jobGraph);

		final SubmittedJobGraph recoveredJobGraph = otherSubmittedJobGraphStore.recoverJobGraph(jobGraph.getJobId());

		assertThat(recoveredJobGraph, is(notNullValue()));

		try {
			otherSubmittedJobGraphStore.removeJobGraph(recoveredJobGraph.getJobId());
			fail("It should not be possible to remove the JobGraph since the first store still has a lock on it.");
		} catch (Exception ignored) {
			// expected
		}

		submittedJobGraphStore.stop();

		// now we should be able to delete the job graph
		otherSubmittedJobGraphStore.removeJobGraph(recoveredJobGraph.getJobId());

		assertThat(otherSubmittedJobGraphStore.recoverJobGraph(recoveredJobGraph.getJobId()), is(nullValue()));

		otherSubmittedJobGraphStore.stop();
	}
}
 
Example #24
Source File: LeaderElectionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void setup() throws Exception {
	try {
		testingServer = new TestingServer();
	} catch (Exception e) {
		throw new RuntimeException("Could not start ZooKeeper testing cluster.", e);
	}

	configuration = new Configuration();

	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

	client = ZooKeeperUtils.startCuratorFramework(configuration);
}
 
Example #25
Source File: ZooKeeperHaServicesTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private void runCleanupTest(
		Configuration configuration,
		TestingBlobStoreService blobStoreService,
		ThrowingConsumer<ZooKeeperHaServices, Exception> zooKeeperHaServicesConsumer) throws Exception {
	try (ZooKeeperHaServices zooKeeperHaServices = new ZooKeeperHaServices(
		ZooKeeperUtils.startCuratorFramework(configuration),
		Executors.directExecutor(),
		configuration,
		blobStoreService)) {

		// create some Zk services to trigger the generation of paths
		final LeaderRetrievalService resourceManagerLeaderRetriever = zooKeeperHaServices.getResourceManagerLeaderRetriever();
		final LeaderElectionService resourceManagerLeaderElectionService = zooKeeperHaServices.getResourceManagerLeaderElectionService();
		final RunningJobsRegistry runningJobsRegistry = zooKeeperHaServices.getRunningJobsRegistry();

		final TestingListener listener = new TestingListener();
		resourceManagerLeaderRetriever.start(listener);
		resourceManagerLeaderElectionService.start(new TestingContender("foobar", resourceManagerLeaderElectionService));
		final JobID jobId = new JobID();
		runningJobsRegistry.setJobRunning(jobId);

		listener.waitForNewLeader(2000L);

		resourceManagerLeaderRetriever.stop();
		resourceManagerLeaderElectionService.stop();
		runningJobsRegistry.clearJob(jobId);

		zooKeeperHaServicesConsumer.accept(zooKeeperHaServices);
	}
}
 
Example #26
Source File: ZooKeeperCheckpointRecoveryFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public CompletedCheckpointStore createCheckpointStore(JobID jobId, int maxNumberOfCheckpointsToRetain, ClassLoader userClassLoader)
		throws Exception {

	return ZooKeeperUtils.createCompletedCheckpoints(client, config, jobId,
			maxNumberOfCheckpointsToRetain, executor);
}
 
Example #27
Source File: ZooKeeperCompletedCheckpointStoreITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected ZooKeeperCompletedCheckpointStore createCompletedCheckpoints(int maxNumberOfCheckpointsToRetain) throws Exception {
	final ZooKeeperStateHandleStore<CompletedCheckpoint> checkpointsInZooKeeper = ZooKeeperUtils.createZooKeeperStateHandleStore(
		ZOOKEEPER.getClient(),
		CHECKPOINT_PATH,
		new TestingRetrievableStateStorageHelper<>());

	return new ZooKeeperCompletedCheckpointStore(
		maxNumberOfCheckpointsToRetain,
		checkpointsInZooKeeper,
		Executors.directExecutor());
}
 
Example #28
Source File: ZooKeeperUtilityFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
public ZooKeeperUtilityFactory(Configuration configuration, String path) throws Exception {
	Preconditions.checkNotNull(path, "path");

	root = ZooKeeperUtils.startCuratorFramework(configuration);

	root.newNamespaceAwareEnsurePath(path).ensure(root.getZookeeperClient());
	facade = root.usingNamespace(ZooKeeperUtils.generateZookeeperPath(root.getNamespace(), path));
}
 
Example #29
Source File: ZooKeeperCompletedCheckpointStoreTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Nonnull
private ZooKeeperCompletedCheckpointStore createZooKeeperCheckpointStore(CuratorFramework client) throws Exception {
	final ZooKeeperStateHandleStore<CompletedCheckpoint> checkpointsInZooKeeper = ZooKeeperUtils.createZooKeeperStateHandleStore(
		client,
		"/checkpoints",
		new TestingRetrievableStateStorageHelper<>());

	return new ZooKeeperCompletedCheckpointStore(
		1,
		checkpointsInZooKeeper,
		Executors.directExecutor());
}
 
Example #30
Source File: ClusterEntrypoint.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Returns the port range for the common {@link RpcService}.
 *
 * @param configuration to extract the port range from
 * @return Port range for the common {@link RpcService}
 */
protected String getRPCPortRange(Configuration configuration) {
	if (ZooKeeperUtils.isZooKeeperRecoveryMode(configuration)) {
		return configuration.getString(HighAvailabilityOptions.HA_JOB_MANAGER_PORT_RANGE);
	} else {
		return String.valueOf(configuration.getInteger(JobManagerOptions.PORT));
	}
}