Java Code Examples for org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService

The following examples show how to use org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: StandaloneUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a {@link StandaloneLeaderRetrievalService} form the given configuration and the
 * JobManager name. The host and port for the remote Akka URL are retrieved from the provided
 * configuration. Instead of using the standard JobManager Akka name, the provided one is used
 * for the remote Akka URL.
 *
 * @param configuration Configuration instance containing hte host and port information
 * @param resolveInitialHostName If true, resolves the hostname of the StandaloneLeaderRetrievalService
 * @param jobManagerName Name of the JobManager actor
 * @return StandaloneLeaderRetrievalService
 * @throws ConfigurationException if the job manager address cannot be retrieved from the configuration
 * @throws UnknownHostException if the job manager address cannot be resolved
 */
public static StandaloneLeaderRetrievalService createLeaderRetrievalService(
		Configuration configuration,
		boolean resolveInitialHostName,
		String jobManagerName)
	throws ConfigurationException, UnknownHostException {
	Tuple2<String, Integer> hostnamePort = HighAvailabilityServicesUtils.getJobManagerAddress(configuration);

	String jobManagerAkkaUrl = AkkaRpcServiceUtils.getRpcUrl(
		hostnamePort.f0,
		hostnamePort.f1,
		jobManagerName != null ? jobManagerName : JobMaster.JOB_MANAGER_NAME,
		resolveInitialHostName ? AddressResolution.TRY_ADDRESS_RESOLUTION : AddressResolution.NO_ADDRESS_RESOLUTION,
		configuration);

	return new StandaloneLeaderRetrievalService(jobManagerAkkaUrl);
}
 
Example 2
/**
 * Tests that the standalone leader election and retrieval service return the same leader
 * URL.
 */
@Test
public void testStandaloneLeaderElectionRetrieval() throws Exception {
	StandaloneLeaderElectionService leaderElectionService = new StandaloneLeaderElectionService();
	StandaloneLeaderRetrievalService leaderRetrievalService = new StandaloneLeaderRetrievalService(TEST_URL);
	TestingContender contender = new TestingContender(TEST_URL, leaderElectionService);
	TestingListener testingListener = new TestingListener();

	try {
		leaderElectionService.start(contender);
		leaderRetrievalService.start(testingListener);

		contender.waitForLeader(1000l);

		assertTrue(contender.isLeader());
		assertEquals(HighAvailabilityServices.DEFAULT_LEADER_ID, contender.getLeaderSessionID());

		testingListener.waitForNewLeader(1000l);

		assertEquals(TEST_URL, testingListener.getAddress());
		assertEquals(HighAvailabilityServices.DEFAULT_LEADER_ID, testingListener.getLeaderSessionID());
	} finally {
		leaderElectionService.stop();
		leaderRetrievalService.stop();
	}
}
 
Example 3
Source Project: flink   Source File: StandaloneUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a {@link StandaloneLeaderRetrievalService} form the given configuration and the
 * JobManager name. The host and port for the remote Akka URL are retrieved from the provided
 * configuration. Instead of using the standard JobManager Akka name, the provided one is used
 * for the remote Akka URL.
 *
 * @param configuration Configuration instance containing hte host and port information
 * @param resolveInitialHostName If true, resolves the hostname of the StandaloneLeaderRetrievalService
 * @param jobManagerName Name of the JobManager actor
 * @return StandaloneLeaderRetrievalService
 * @throws ConfigurationException if the job manager address cannot be retrieved from the configuration
 * @throws UnknownHostException if the job manager address cannot be resolved
 */
public static StandaloneLeaderRetrievalService createLeaderRetrievalService(
		Configuration configuration,
		boolean resolveInitialHostName,
		String jobManagerName)
	throws ConfigurationException, UnknownHostException {
	Tuple2<String, Integer> hostnamePort = HighAvailabilityServicesUtils.getJobManagerAddress(configuration);

	String jobManagerAkkaUrl = AkkaRpcServiceUtils.getRpcUrl(
		hostnamePort.f0,
		hostnamePort.f1,
		jobManagerName != null ? jobManagerName : JobMaster.JOB_MANAGER_NAME,
		resolveInitialHostName ? AddressResolution.TRY_ADDRESS_RESOLUTION : AddressResolution.NO_ADDRESS_RESOLUTION,
		configuration);

	return new StandaloneLeaderRetrievalService(jobManagerAkkaUrl);
}
 
Example 4
Source Project: flink   Source File: StandaloneLeaderElectionTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the standalone leader election and retrieval service return the same leader
 * URL.
 */
@Test
public void testStandaloneLeaderElectionRetrieval() throws Exception {
	StandaloneLeaderElectionService leaderElectionService = new StandaloneLeaderElectionService();
	StandaloneLeaderRetrievalService leaderRetrievalService = new StandaloneLeaderRetrievalService(TEST_URL);
	TestingContender contender = new TestingContender(TEST_URL, leaderElectionService);
	TestingListener testingListener = new TestingListener();

	try {
		leaderElectionService.start(contender);
		leaderRetrievalService.start(testingListener);

		contender.waitForLeader(1000l);

		assertTrue(contender.isLeader());
		assertEquals(HighAvailabilityServices.DEFAULT_LEADER_ID, contender.getLeaderSessionID());

		testingListener.waitForNewLeader(1000l);

		assertEquals(TEST_URL, testingListener.getAddress());
		assertEquals(HighAvailabilityServices.DEFAULT_LEADER_ID, testingListener.getLeaderSessionID());
	} finally {
		leaderElectionService.stop();
		leaderRetrievalService.stop();
	}
}
 
Example 5
Source Project: flink   Source File: StandaloneLeaderElectionTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the standalone leader election and retrieval service return the same leader
 * URL.
 */
@Test
public void testStandaloneLeaderElectionRetrieval() throws Exception {
	StandaloneLeaderElectionService leaderElectionService = new StandaloneLeaderElectionService();
	StandaloneLeaderRetrievalService leaderRetrievalService = new StandaloneLeaderRetrievalService(TEST_URL);
	TestingContender contender = new TestingContender(TEST_URL, leaderElectionService);
	TestingListener testingListener = new TestingListener();

	try {
		leaderElectionService.start(contender);
		leaderRetrievalService.start(testingListener);

		contender.waitForLeader(1000l);

		assertTrue(contender.isLeader());
		assertEquals(HighAvailabilityServices.DEFAULT_LEADER_ID, contender.getLeaderSessionID());

		testingListener.waitForNewLeader(1000l);

		assertEquals(TEST_URL, testingListener.getAddress());
		assertEquals(HighAvailabilityServices.DEFAULT_LEADER_ID, testingListener.getLeaderSessionID());
	} finally {
		leaderElectionService.stop();
		leaderRetrievalService.stop();
	}
}
 
Example 6
@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	enter();
	try {
		return new StandaloneLeaderRetrievalService(resourceManagerRpcUrl, DEFAULT_LEADER_ID);
	}
	finally {
		exit();
	}
}
 
Example 7
@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	enter();

	try {
		return new StandaloneLeaderRetrievalService(dispatcherRpcUrl, DEFAULT_LEADER_ID);
	} finally {
		exit();
	}
}
 
Example 8
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
	enter();
	try {
		return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID);
	} finally {
		exit();
	}
}
 
Example 9
Source Project: Flink-CEPplus   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(resourceManagerAddress, DEFAULT_LEADER_ID);
	}

}
 
Example 10
Source Project: Flink-CEPplus   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(dispatcherAddress, DEFAULT_LEADER_ID);
	}
}
 
Example 11
Source Project: Flink-CEPplus   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(jobManagerAddress, DEFAULT_LEADER_ID);
	}
}
 
Example 12
Source Project: Flink-CEPplus   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID);
	}
}
 
Example 13
Source Project: Flink-CEPplus   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getWebMonitorLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(webMonitorAddress, DEFAULT_LEADER_ID);
	}
}
 
Example 14
@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	enter();
	try {
		return new StandaloneLeaderRetrievalService(resourceManagerRpcUrl, DEFAULT_LEADER_ID);
	}
	finally {
		exit();
	}
}
 
Example 15
@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	enter();

	try {
		return new StandaloneLeaderRetrievalService(dispatcherRpcUrl, DEFAULT_LEADER_ID);
	} finally {
		exit();
	}
}
 
Example 16
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
	enter();
	try {
		return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID);
	} finally {
		exit();
	}
}
 
Example 17
Source Project: flink   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(resourceManagerAddress, DEFAULT_LEADER_ID);
	}

}
 
Example 18
Source Project: flink   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(dispatcherAddress, DEFAULT_LEADER_ID);
	}
}
 
Example 19
Source Project: flink   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(jobManagerAddress, DEFAULT_LEADER_ID);
	}
}
 
Example 20
Source Project: flink   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID);
	}
}
 
Example 21
Source Project: flink   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getWebMonitorLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(webMonitorAddress, DEFAULT_LEADER_ID);
	}
}
 
Example 22
Source Project: flink   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(resourceManagerAddress, DEFAULT_LEADER_ID);
	}

}
 
Example 23
Source Project: flink   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(dispatcherAddress, DEFAULT_LEADER_ID);
	}
}
 
Example 24
Source Project: flink   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService("UNKNOWN", DEFAULT_LEADER_ID);
	}
}
 
Example 25
Source Project: flink   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID);
	}
}
 
Example 26
Source Project: flink   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getClusterRestEndpointLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(clusterRestEndpointAddress, DEFAULT_LEADER_ID);
	}
}
 
Example 27
Source Project: flink   Source File: StandaloneClientHAServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getClusterRestEndpointLeaderRetriever() {
	synchronized (lock) {
		checkState(running, "ClientHaService has already been closed.");
		return new StandaloneLeaderRetrievalService(webMonitorAddress, DEFAULT_LEADER_ID);
	}
}
 
Example 28
Source Project: Flink-CEPplus   Source File: TaskManagerTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 *  Tests that repeated local {@link PartitionNotFoundException}s ultimately fail the receiver.
 */
@Test
public void testLocalPartitionNotFound() throws Exception {

	new JavaTestKit(system){{

		ActorGateway jobManager = null;
		ActorGateway taskManager = null;

		final ActorGateway testActorGateway = new AkkaActorGateway(
				getTestActor(),
			LEADER_SESSION_ID);

		try {
			final IntermediateDataSetID resultId = new IntermediateDataSetID();

			// Create the JM
			ActorRef jm = system.actorOf(Props.create(
					new SimplePartitionStateLookupJobManagerCreator(LEADER_SESSION_ID, getTestActor())));

			jobManager = new AkkaActorGateway(jm, LEADER_SESSION_ID);

			highAvailabilityServices.setJobMasterLeaderRetriever(
				HighAvailabilityServices.DEFAULT_JOB_ID,
				new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));

			final Configuration config = new Configuration();
			config.setInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_INITIAL, 100);
			config.setInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_MAX, 200);

			taskManager = TestingUtils.createTaskManager(
					system,
					highAvailabilityServices,
					config,
					true,
					true);

			// ---------------------------------------------------------------------------------

			final ActorGateway tm = taskManager;

			final JobID jid = new JobID();
			final JobVertexID vid = new JobVertexID();
			final ExecutionAttemptID eid = new ExecutionAttemptID();

			final ResultPartitionID partitionId = new ResultPartitionID();

			// Local location (on the same TM though) for the partition
			final ResultPartitionLocation loc = ResultPartitionLocation.createLocal();

			final InputChannelDeploymentDescriptor[] icdd =
					new InputChannelDeploymentDescriptor[] {
							new InputChannelDeploymentDescriptor(partitionId, loc)};

			final InputGateDeploymentDescriptor igdd =
					new InputGateDeploymentDescriptor(resultId, ResultPartitionType.PIPELINED, 0, icdd);

			final TaskDeploymentDescriptor tdd = createTaskDeploymentDescriptor(
					jid, "TestJob", vid, eid,
					new SerializedValue<>(new ExecutionConfig()),
					"Receiver", 1, 0, 1, 0,
					new Configuration(), new Configuration(),
					Tasks.AgnosticReceiver.class.getName(),
					Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
					Collections.singletonList(igdd),
					Collections.emptyList(),
					Collections.emptyList(), 0);

			new Within(new FiniteDuration(120, TimeUnit.SECONDS)) {
				@Override
				protected void run() {
					// Submit the task
					tm.tell(new SubmitTask(tdd), testActorGateway);
					expectMsgClass(Acknowledge.get().getClass());

					// Wait to be notified about the final execution state by the mock JM
					TaskExecutionState msg = expectMsgClass(TaskExecutionState.class);

					// The task should fail after repeated requests
					assertEquals(msg.getExecutionState(), ExecutionState.FAILED);

					Throwable error = msg.getError(getClass().getClassLoader());
					if (error.getClass() != PartitionNotFoundException.class) {
						error.printStackTrace();
						fail("Wrong exception: " + error.getMessage());
					}
				}
			};
		}
		catch (Exception e) {
			e.printStackTrace();
			fail(e.getMessage());
		}
		finally {
			TestingUtils.stopActor(taskManager);
			TestingUtils.stopActor(jobManager);
		}
	}};
}
 
Example 29
Source Project: Flink-CEPplus   Source File: TaskManagerTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testLogNotFoundHandling() throws Exception {

	new JavaTestKit(system){{

		// we require a JobManager so that the BlobService is also started
		ActorGateway jobManager = null;
		ActorGateway taskManager = null;

		try {

			// Create the JM
			ActorRef jm = system.actorOf(Props.create(
				new SimplePartitionStateLookupJobManagerCreator(LEADER_SESSION_ID, getTestActor())));

			jobManager = new AkkaActorGateway(jm, LEADER_SESSION_ID);

			final int dataPort = NetUtils.getAvailablePort();
			Configuration config = new Configuration();
			config.setInteger(TaskManagerOptions.DATA_PORT, dataPort);
			config.setInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_INITIAL, 100);
			config.setInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_MAX, 200);
			config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, "/i/dont/exist");

			highAvailabilityServices.setJobMasterLeaderRetriever(
				HighAvailabilityServices.DEFAULT_JOB_ID,
				new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));

			taskManager = TestingUtils.createTaskManager(
				system,
				highAvailabilityServices,
				config,
				false,
				true);

			// ---------------------------------------------------------------------------------

			final ActorGateway tm = taskManager;

			new Within(d) {
				@Override
				protected void run() {
					Future<Object> logFuture = tm.ask(TaskManagerMessages.getRequestTaskManagerLog(), timeout);
					try {
						Await.result(logFuture, timeout);
						Assert.fail();
					} catch (Exception e) {
						Assert.assertTrue(e.getMessage().startsWith("TaskManager log files are unavailable. Log file could not be found at"));
					}
				}
			};
		} finally {
			TestingUtils.stopActor(taskManager);
			TestingUtils.stopActor(jobManager);
		}
	}};
}
 
Example 30
Source Project: Flink-CEPplus   Source File: TaskManagerTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Test that a failing schedule or update consumers call leads to the failing of the respective
 * task.
 *
 * <p>IMPORTANT: We have to make sure that the invokable's cancel method is called, because only
 * then the future is completed. We do this by not eagerly deploy consumer tasks and requiring
 * the invokable to fill one memory segment. The completed memory segment will trigger the
 * scheduling of the downstream operator since it is in pipeline mode. After we've filled the
 * memory segment, we'll block the invokable and wait for the task failure due to the failed
 * schedule or update consumers call.
 */
@Test(timeout = 10000L)
public void testFailingScheduleOrUpdateConsumersMessage() throws Exception {
	new JavaTestKit(system) {{
		final Configuration configuration = new Configuration();

		// set the memory segment to the smallest size possible, because we have to fill one
		// memory buffer to trigger the schedule or update consumers message to the downstream
		// operators
		configuration.setString(TaskManagerOptions.MEMORY_SEGMENT_SIZE, "4096");

		final JobID jid = new JobID();
		final JobVertexID vid = new JobVertexID();
		final ExecutionAttemptID eid = new ExecutionAttemptID();
		final SerializedValue<ExecutionConfig> executionConfig = new SerializedValue<>(new ExecutionConfig());

		final ResultPartitionDeploymentDescriptor resultPartitionDeploymentDescriptor = new ResultPartitionDeploymentDescriptor(
			new IntermediateDataSetID(),
			new IntermediateResultPartitionID(),
			ResultPartitionType.PIPELINED,
			1,
			1,
			true);

		final TaskDeploymentDescriptor tdd = createTaskDeploymentDescriptor(jid, "TestJob", vid, eid, executionConfig,
			"TestTask", 1, 0, 1, 0, new Configuration(), new Configuration(),
			TestInvokableRecordCancel.class.getName(),
			Collections.singletonList(resultPartitionDeploymentDescriptor),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			new ArrayList<>(), Collections.emptyList(), 0);

		ActorRef jmActorRef = system.actorOf(Props.create(FailingScheduleOrUpdateConsumersJobManager.class, LEADER_SESSION_ID), "jobmanager");
		ActorGateway jobManager = new AkkaActorGateway(jmActorRef, LEADER_SESSION_ID);

		highAvailabilityServices.setJobMasterLeaderRetriever(
			HighAvailabilityServices.DEFAULT_JOB_ID,
			new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));

		final ActorGateway taskManager = TestingUtils.createTaskManager(
			system,
			highAvailabilityServices,
			configuration,
			true,
			true);

		try {
			TestInvokableRecordCancel.resetGotCanceledFuture();

			Future<Object> result = taskManager.ask(new SubmitTask(tdd), timeout);

			Await.result(result, timeout);

			CompletableFuture<Boolean> cancelFuture = TestInvokableRecordCancel.gotCanceled();

			assertEquals(true, cancelFuture.get());
		} finally {
			TestingUtils.stopActor(taskManager);
			TestingUtils.stopActor(jobManager);
		}
	}};
}