Java Code Examples for org.apache.flink.runtime.util.ZooKeeperUtils#createLeaderRetrievalService()

The following examples show how to use org.apache.flink.runtime.util.ZooKeeperUtils#createLeaderRetrievalService() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: ZooKeeperLeaderElectionTest.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Tests that the ZooKeeperLeaderElection/RetrievalService return both the correct URL.
 */
@Test
public void testZooKeeperLeaderElectionRetrieval() throws Exception {
	ZooKeeperLeaderElectionService leaderElectionService = null;
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	try {
		leaderElectionService = ZooKeeperUtils.createLeaderElectionService(client, configuration);
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		TestingContender contender = new TestingContender(TEST_URL, leaderElectionService);
		TestingListener listener = new TestingListener();

		leaderElectionService.start(contender);
		leaderRetrievalService.start(listener);

		contender.waitForLeader(timeout);

		assertTrue(contender.isLeader());
		assertEquals(leaderElectionService.getLeaderSessionID(), contender.getLeaderSessionID());

		listener.waitForNewLeader(timeout);

		assertEquals(TEST_URL, listener.getAddress());
		assertEquals(leaderElectionService.getLeaderSessionID(), listener.getLeaderSessionID());

	} finally {
		if (leaderElectionService != null) {
			leaderElectionService.stop();
		}

		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}
	}
}

Example 2

Source File: ZooKeeperLeaderElectionTest.java From flink with Apache License 2.0

5 votes

/**
 * Tests that the ZooKeeperLeaderElection/RetrievalService return both the correct URL.
 */
@Test
public void testZooKeeperLeaderElectionRetrieval() throws Exception {
	ZooKeeperLeaderElectionService leaderElectionService = null;
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	try {
		leaderElectionService = ZooKeeperUtils.createLeaderElectionService(client, configuration);
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		TestingContender contender = new TestingContender(TEST_URL, leaderElectionService);
		TestingListener listener = new TestingListener();

		leaderElectionService.start(contender);
		leaderRetrievalService.start(listener);

		contender.waitForLeader(timeout);

		assertTrue(contender.isLeader());
		assertEquals(leaderElectionService.getLeaderSessionID(), contender.getLeaderSessionID());

		listener.waitForNewLeader(timeout);

		assertEquals(TEST_URL, listener.getAddress());
		assertEquals(leaderElectionService.getLeaderSessionID(), listener.getLeaderSessionID());

	} finally {
		if (leaderElectionService != null) {
			leaderElectionService.stop();
		}

		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}
	}
}

Example 3

Source File: ZooKeeperHaServices.java From flink with Apache License 2.0

4 votes

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, getPathForJobManager(jobID));
}

Example 4

Source File: ZooKeeperHaServices.java From flink with Apache License 2.0

4 votes

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, getPathForJobManager(jobID));
}

Example 5

Source File: ZooKeeperHaServices.java From flink with Apache License 2.0

4 votes

@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, DISPATCHER_LEADER_PATH);
}

Example 6

Source File: ZooKeeperHaServices.java From flink with Apache License 2.0

4 votes

@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, RESOURCE_MANAGER_LEADER_PATH);
}

Example 7

Source File: ZooKeeperClientHAServices.java From flink with Apache License 2.0

4 votes

@Override
public LeaderRetrievalService getClusterRestEndpointLeaderRetriever() {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, REST_SERVER_LEADER_PATH);
}

Example 8

Source File: ZooKeeperLeaderElectionTest.java From flink with Apache License 2.0

4 votes

/**
 *  Test that errors in the {@link LeaderElectionService} are correctly forwarded to the
 *  {@link LeaderContender}.
 */
@Test
public void testExceptionForwarding() throws Exception {
	ZooKeeperLeaderElectionService leaderElectionService = null;
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;
	TestingListener listener = new TestingListener();
	TestingContender testingContender;

	CuratorFramework client;
	final CreateBuilder mockCreateBuilder = mock(CreateBuilder.class, Mockito.RETURNS_DEEP_STUBS);
	final Exception testException = new Exception("Test exception");

	try {
		client = spy(ZooKeeperUtils.startCuratorFramework(configuration));

		Answer<CreateBuilder> answer = new Answer<CreateBuilder>() {
			private int counter = 0;

			@Override
			public CreateBuilder answer(InvocationOnMock invocation) throws Throwable {
				counter++;

				// at first we have to create the leader latch, there it mustn't fail yet
				if (counter < 2) {
					return (CreateBuilder) invocation.callRealMethod();
				} else {
					return mockCreateBuilder;
				}
			}
		};

		doAnswer(answer).when(client).create();

		when(
			mockCreateBuilder
			.creatingParentsIfNeeded()
			.withMode(Matchers.any(CreateMode.class))
			.forPath(anyString(), any(byte[].class))).thenThrow(testException);

		leaderElectionService = new ZooKeeperLeaderElectionService(client, "/latch", "/leader");
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		testingContender = new TestingContender(TEST_URL, leaderElectionService);

		leaderElectionService.start(testingContender);
		leaderRetrievalService.start(listener);

		testingContender.waitForError(timeout);

		assertNotNull(testingContender.getError());
		assertEquals(testException, testingContender.getError().getCause());
	} finally {
		if (leaderElectionService != null) {
			leaderElectionService.stop();
		}

		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}
	}
}

Example 9

Source File: ZooKeeperLeaderElectionTest.java From flink with Apache License 2.0

4 votes

/**
 * Tests the repeated reelection of {@link LeaderContender} once the current leader dies.
 * Furthermore, it tests that new LeaderElectionServices can be started later on and that they
 * successfully register at ZooKeeper and take part in the leader election.
 */
@Test
public void testZooKeeperReelectionWithReplacement() throws Exception {
	int num = 3;
	int numTries = 30;

	ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
	TestingContender[] contenders = new TestingContender[num];
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	TestingListener listener = new TestingListener();

	try {
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		leaderRetrievalService.start(listener);

		for (int i = 0; i < num; i++) {
			leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
			contenders[i] = new TestingContender(TEST_URL + "_" + i + "_0", leaderElectionService[i]);

			leaderElectionService[i].start(contenders[i]);
		}

		String pattern = TEST_URL + "_" + "(\\d+)" + "_" + "(\\d+)";
		Pattern regex = Pattern.compile(pattern);

		for (int i = 0; i < numTries; i++) {
			listener.waitForNewLeader(timeout);

			String address = listener.getAddress();

			Matcher m = regex.matcher(address);

			if (m.find()) {
				int index = Integer.parseInt(m.group(1));
				int lastTry = Integer.parseInt(m.group(2));

				assertEquals(listener.getLeaderSessionID(), contenders[index].getLeaderSessionID());

				// stop leader election service = revoke leadership
				leaderElectionService[index].stop();
				// create new leader election service which takes part in the leader election
				leaderElectionService[index] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
				contenders[index] = new TestingContender(
						TEST_URL + "_" + index + "_" + (lastTry + 1),
						leaderElectionService[index]);

				leaderElectionService[index].start(contenders[index]);
			} else {
				throw new Exception("Did not find the leader's index.");
			}
		}

	} finally {
		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}

		for (ZooKeeperLeaderElectionService electionService : leaderElectionService) {
			if (electionService != null) {
				electionService.stop();
			}
		}
	}
}

Example 10

Source File: ZooKeeperLeaderElectionTest.java From flink with Apache License 2.0

4 votes

/**
 * Tests repeatedly the reelection of still available LeaderContender. After a contender has
 * been elected as the leader, it is removed. This forces the ZooKeeperLeaderElectionService
 * to elect a new leader.
 */
@Test
public void testZooKeeperReelection() throws Exception {
	Deadline deadline = Deadline.fromNow(Duration.ofMinutes(5L));

	int num = 10;

	ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
	TestingContender[] contenders = new TestingContender[num];
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	TestingListener listener = new TestingListener();

	try {
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		LOG.debug("Start leader retrieval service for the TestingListener.");

		leaderRetrievalService.start(listener);

		for (int i = 0; i < num; i++) {
			leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
			contenders[i] = new TestingContender(TEST_URL + "_" + i, leaderElectionService[i]);

			LOG.debug("Start leader election service for contender #{}.", i);

			leaderElectionService[i].start(contenders[i]);
		}

		String pattern = TEST_URL + "_" + "(\\d+)";
		Pattern regex = Pattern.compile(pattern);

		int numberSeenLeaders = 0;

		while (deadline.hasTimeLeft() && numberSeenLeaders < num) {
			LOG.debug("Wait for new leader #{}.", numberSeenLeaders);
			String address = listener.waitForNewLeader(deadline.timeLeft().toMillis());

			Matcher m = regex.matcher(address);

			if (m.find()) {
				int index = Integer.parseInt(m.group(1));

				TestingContender contender = contenders[index];

				// check that the retrieval service has retrieved the correct leader
				if (address.equals(contender.getAddress()) && listener.getLeaderSessionID().equals(contender.getLeaderSessionID())) {
					// kill the election service of the leader
					LOG.debug("Stop leader election service of contender #{}.", numberSeenLeaders);
					leaderElectionService[index].stop();
					leaderElectionService[index] = null;

					numberSeenLeaders++;
				}
			} else {
				fail("Did not find the leader's index.");
			}
		}

		assertFalse("Did not complete the leader reelection in time.", deadline.isOverdue());
		assertEquals(num, numberSeenLeaders);

	} finally {
		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}

		for (ZooKeeperLeaderElectionService electionService : leaderElectionService) {
			if (electionService != null) {
				electionService.stop();
			}
		}
	}
}

Example 11

Source File: ZooKeeperHaServices.java From flink with Apache License 2.0

4 votes

@Override
public LeaderRetrievalService getWebMonitorLeaderRetriever() {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, REST_SERVER_LEADER_PATH);
}

Example 12

Source File: ZooKeeperLeaderElectionTest.java From flink with Apache License 2.0

4 votes

/**
 * Tests repeatedly the reelection of still available LeaderContender. After a contender has
 * been elected as the leader, it is removed. This forces the ZooKeeperLeaderElectionService
 * to elect a new leader.
 */
@Test
public void testZooKeeperReelection() throws Exception {
	Deadline deadline = Deadline.fromNow(Duration.ofMinutes(5L));

	int num = 10;

	ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
	TestingContender[] contenders = new TestingContender[num];
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	TestingListener listener = new TestingListener();

	try {
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		LOG.debug("Start leader retrieval service for the TestingListener.");

		leaderRetrievalService.start(listener);

		for (int i = 0; i < num; i++) {
			leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
			contenders[i] = new TestingContender(createAddress(i), leaderElectionService[i]);

			LOG.debug("Start leader election service for contender #{}.", i);

			leaderElectionService[i].start(contenders[i]);
		}

		String pattern = TEST_URL + "_" + "(\\d+)";
		Pattern regex = Pattern.compile(pattern);

		int numberSeenLeaders = 0;

		while (deadline.hasTimeLeft() && numberSeenLeaders < num) {
			LOG.debug("Wait for new leader #{}.", numberSeenLeaders);
			String address = listener.waitForNewLeader(deadline.timeLeft().toMillis());

			Matcher m = regex.matcher(address);

			if (m.find()) {
				int index = Integer.parseInt(m.group(1));

				TestingContender contender = contenders[index];

				// check that the retrieval service has retrieved the correct leader
				if (address.equals(createAddress(index)) && listener.getLeaderSessionID().equals(contender.getLeaderSessionID())) {
					// kill the election service of the leader
					LOG.debug("Stop leader election service of contender #{}.", numberSeenLeaders);
					leaderElectionService[index].stop();
					leaderElectionService[index] = null;

					numberSeenLeaders++;
				}
			} else {
				fail("Did not find the leader's index.");
			}
		}

		assertFalse("Did not complete the leader reelection in time.", deadline.isOverdue());
		assertEquals(num, numberSeenLeaders);

	} finally {
		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}

		for (ZooKeeperLeaderElectionService electionService : leaderElectionService) {
			if (electionService != null) {
				electionService.stop();
			}
		}
	}
}

Example 13

Source File: ZooKeeperHaServices.java From flink with Apache License 2.0

4 votes

@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, DISPATCHER_LEADER_PATH);
}

Example 14

Source File: ZooKeeperHaServices.java From flink with Apache License 2.0

4 votes

@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, RESOURCE_MANAGER_LEADER_PATH);
}

Example 15

Source File: ZooKeeperLeaderElectionTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 *  Test that errors in the {@link LeaderElectionService} are correctly forwarded to the
 *  {@link LeaderContender}.
 */
@Test
public void testExceptionForwarding() throws Exception {
	ZooKeeperLeaderElectionService leaderElectionService = null;
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;
	TestingListener listener = new TestingListener();
	TestingContender testingContender;

	CuratorFramework client;
	final CreateBuilder mockCreateBuilder = mock(CreateBuilder.class, Mockito.RETURNS_DEEP_STUBS);
	final Exception testException = new Exception("Test exception");

	try {
		client = spy(ZooKeeperUtils.startCuratorFramework(configuration));

		Answer<CreateBuilder> answer = new Answer<CreateBuilder>() {
			private int counter = 0;

			@Override
			public CreateBuilder answer(InvocationOnMock invocation) throws Throwable {
				counter++;

				// at first we have to create the leader latch, there it mustn't fail yet
				if (counter < 2) {
					return (CreateBuilder) invocation.callRealMethod();
				} else {
					return mockCreateBuilder;
				}
			}
		};

		doAnswer(answer).when(client).create();

		when(
			mockCreateBuilder
			.creatingParentsIfNeeded()
			.withMode(Matchers.any(CreateMode.class))
			.forPath(anyString(), any(byte[].class))).thenThrow(testException);

		leaderElectionService = new ZooKeeperLeaderElectionService(client, "/latch", "/leader");
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		testingContender = new TestingContender(TEST_URL, leaderElectionService);

		leaderElectionService.start(testingContender);
		leaderRetrievalService.start(listener);

		testingContender.waitForError(timeout);

		assertNotNull(testingContender.getError());
		assertEquals(testException, testingContender.getError().getCause());
	} finally {
		if (leaderElectionService != null) {
			leaderElectionService.stop();
		}

		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}
	}
}

Example 16

Source File: ZooKeeperLeaderElectionTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Tests the repeated reelection of {@link LeaderContender} once the current leader dies.
 * Furthermore, it tests that new LeaderElectionServices can be started later on and that they
 * successfully register at ZooKeeper and take part in the leader election.
 */
@Test
public void testZooKeeperReelectionWithReplacement() throws Exception {
	int num = 3;
	int numTries = 30;

	ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
	TestingContender[] contenders = new TestingContender[num];
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	TestingListener listener = new TestingListener();

	try {
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		leaderRetrievalService.start(listener);

		for (int i = 0; i < num; i++) {
			leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
			contenders[i] = new TestingContender(TEST_URL + "_" + i + "_0", leaderElectionService[i]);

			leaderElectionService[i].start(contenders[i]);
		}

		String pattern = TEST_URL + "_" + "(\\d+)" + "_" + "(\\d+)";
		Pattern regex = Pattern.compile(pattern);

		for (int i = 0; i < numTries; i++) {
			listener.waitForNewLeader(timeout);

			String address = listener.getAddress();

			Matcher m = regex.matcher(address);

			if (m.find()) {
				int index = Integer.parseInt(m.group(1));
				int lastTry = Integer.parseInt(m.group(2));

				assertEquals(listener.getLeaderSessionID(), contenders[index].getLeaderSessionID());

				// stop leader election service = revoke leadership
				leaderElectionService[index].stop();
				// create new leader election service which takes part in the leader election
				leaderElectionService[index] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
				contenders[index] = new TestingContender(
						TEST_URL + "_" + index + "_" + (lastTry + 1),
						leaderElectionService[index]);

				leaderElectionService[index].start(contenders[index]);
			} else {
				throw new Exception("Did not find the leader's index.");
			}
		}

	} finally {
		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}

		for (ZooKeeperLeaderElectionService electionService : leaderElectionService) {
			if (electionService != null) {
				electionService.stop();
			}
		}
	}
}

Example 17

Source File: ZooKeeperLeaderElectionTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Tests repeatedly the reelection of still available LeaderContender. After a contender has
 * been elected as the leader, it is removed. This forces the ZooKeeperLeaderElectionService
 * to elect a new leader.
 */
@Test
public void testZooKeeperReelection() throws Exception {
	Deadline deadline = Deadline.fromNow(Duration.ofMinutes(5L));

	int num = 10;

	ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
	TestingContender[] contenders = new TestingContender[num];
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	TestingListener listener = new TestingListener();

	try {
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		LOG.debug("Start leader retrieval service for the TestingListener.");

		leaderRetrievalService.start(listener);

		for (int i = 0; i < num; i++) {
			leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration);
			contenders[i] = new TestingContender(TEST_URL + "_" + i, leaderElectionService[i]);

			LOG.debug("Start leader election service for contender #{}.", i);

			leaderElectionService[i].start(contenders[i]);
		}

		String pattern = TEST_URL + "_" + "(\\d+)";
		Pattern regex = Pattern.compile(pattern);

		int numberSeenLeaders = 0;

		while (deadline.hasTimeLeft() && numberSeenLeaders < num) {
			LOG.debug("Wait for new leader #{}.", numberSeenLeaders);
			String address = listener.waitForNewLeader(deadline.timeLeft().toMillis());

			Matcher m = regex.matcher(address);

			if (m.find()) {
				int index = Integer.parseInt(m.group(1));

				TestingContender contender = contenders[index];

				// check that the retrieval service has retrieved the correct leader
				if (address.equals(contender.getAddress()) && listener.getLeaderSessionID().equals(contender.getLeaderSessionID())) {
					// kill the election service of the leader
					LOG.debug("Stop leader election service of contender #{}.", numberSeenLeaders);
					leaderElectionService[index].stop();
					leaderElectionService[index] = null;

					numberSeenLeaders++;
				}
			} else {
				fail("Did not find the leader's index.");
			}
		}

		assertFalse("Did not complete the leader reelection in time.", deadline.isOverdue());
		assertEquals(num, numberSeenLeaders);

	} finally {
		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}

		for (ZooKeeperLeaderElectionService electionService : leaderElectionService) {
			if (electionService != null) {
				electionService.stop();
			}
		}
	}
}

Example 18

Source File: ZooKeeperHaServices.java From flink with Apache License 2.0

4 votes

@Override
public LeaderRetrievalService getClusterRestEndpointLeaderRetriever() {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, REST_SERVER_LEADER_PATH);
}

Example 19

Source File: ZooKeeperHaServices.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, getPathForJobManager(jobID));
}

Example 20

Source File: ZooKeeperHaServices.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, DISPATCHER_LEADER_PATH);
}