org.apache.flink.mesos.util.MesosConfiguration Java Examples

The following examples show how to use org.apache.flink.mesos.util.MesosConfiguration. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: LaunchableMesosWorker.java From flink with Apache License 2.0

5 votes

/**
 * Construct a launchable Mesos worker.
 * @param resolver The resolver for retrieving artifacts (e.g. jars, configuration)
 * @param params the TM parameters such as memory, cpu to acquire.
 * @param containerSpec an abstract container specification for launch time.
 * @param taskID the taskID for this worker.
 */
public LaunchableMesosWorker(
		MesosArtifactResolver resolver,
		MesosTaskManagerParameters params,
		ContainerSpecification containerSpec,
		Protos.TaskID taskID,
		MesosConfiguration mesosConfiguration) {
	this.resolver = Preconditions.checkNotNull(resolver);
	this.containerSpec = Preconditions.checkNotNull(containerSpec);
	this.params = Preconditions.checkNotNull(params);
	this.taskID = Preconditions.checkNotNull(taskID);
	this.mesosConfiguration = Preconditions.checkNotNull(mesosConfiguration);

	this.taskRequest = new Request();
}

Example #2

Source File: MesosResourceManagerTest.java From flink with Apache License 2.0

5 votes

MockMesosResourceManagerRuntimeServices() throws Exception {
	schedulerDriver = mock(SchedulerDriver.class);

	mesosConfig = mock(MesosConfiguration.class);
	when(mesosConfig.frameworkInfo()).thenReturn(Protos.FrameworkInfo.newBuilder());
	when(mesosConfig.withFrameworkInfo(any(Protos.FrameworkInfo.Builder.class))).thenReturn(mesosConfig);
	when(mesosConfig.createDriver(any(Scheduler.class), anyBoolean())).thenReturn(schedulerDriver);

	workerStore = mock(MesosWorkerStore.class);
	when(workerStore.getFrameworkID()).thenReturn(Option.<Protos.FrameworkID>empty());

	artifactServer = mock(MesosArtifactServer.class);
}

Example #3

Source File: MesosResourceManagerTest.java From flink with Apache License 2.0

5 votes

public TestingMesosResourceManager(
	RpcService rpcService,
	String resourceManagerEndpointId,
	ResourceID resourceId,
	HighAvailabilityServices highAvailabilityServices,
	HeartbeatServices heartbeatServices,
	SlotManager slotManager,
	JobLeaderIdService jobLeaderIdService,
	FatalErrorHandler fatalErrorHandler,

	// Mesos specifics
	Configuration flinkConfig,
	MesosServices mesosServices,
	MesosConfiguration mesosConfig,
	MesosTaskManagerParameters taskManagerParameters,
	ContainerSpecification taskManagerContainerSpec,
	ResourceManagerMetricGroup resourceManagerMetricGroup) {
	super(
		rpcService,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		NoOpResourceManagerPartitionTracker::get,
		jobLeaderIdService,
		new ClusterInformation("localhost", 1234),
		fatalErrorHandler,
		flinkConfig,
		mesosServices,
		mesosConfig,
		taskManagerParameters,
		taskManagerContainerSpec,
		null,
		resourceManagerMetricGroup);
}

Example #4

Source File: LaunchableMesosWorker.java From flink with Apache License 2.0

5 votes

/**
 * Construct a launchable Mesos worker.
 * @param resolver The resolver for retrieving artifacts (e.g. jars, configuration)
 * @param params the TM parameters such as memory, cpu to acquire.
 * @param containerSpec an abstract container specification for launch time.
 * @param taskID the taskID for this worker.
 */
public LaunchableMesosWorker(
		MesosArtifactResolver resolver,
		MesosTaskManagerParameters params,
		ContainerSpecification containerSpec,
		Protos.TaskID taskID,
		MesosConfiguration mesosConfiguration) {
	this.resolver = Preconditions.checkNotNull(resolver);
	this.containerSpec = Preconditions.checkNotNull(containerSpec);
	this.params = Preconditions.checkNotNull(params);
	this.taskID = Preconditions.checkNotNull(taskID);
	this.mesosConfiguration = Preconditions.checkNotNull(mesosConfiguration);

	this.taskRequest = new Request();
}

Example #5

Source File: MesosResourceManagerTest.java From flink with Apache License 2.0

5 votes

MockMesosResourceManagerRuntimeServices() throws Exception {
	schedulerDriver = mock(SchedulerDriver.class);

	mesosConfig = mock(MesosConfiguration.class);
	when(mesosConfig.frameworkInfo()).thenReturn(Protos.FrameworkInfo.newBuilder());
	when(mesosConfig.withFrameworkInfo(any(Protos.FrameworkInfo.Builder.class))).thenReturn(mesosConfig);
	when(mesosConfig.createDriver(any(Scheduler.class), anyBoolean())).thenReturn(schedulerDriver);

	workerStore = mock(MesosWorkerStore.class);
	when(workerStore.getFrameworkID()).thenReturn(Option.<Protos.FrameworkID>empty());

	artifactServer = mock(MesosArtifactServer.class);
}

Example #6

Source File: MesosResourceManagerTest.java From flink with Apache License 2.0

5 votes

public TestingMesosResourceManager(
	RpcService rpcService,
	String resourceManagerEndpointId,
	ResourceID resourceId,
	HighAvailabilityServices highAvailabilityServices,
	HeartbeatServices heartbeatServices,
	SlotManager slotManager,
	MetricRegistry metricRegistry,
	JobLeaderIdService jobLeaderIdService,
	FatalErrorHandler fatalErrorHandler,

	// Mesos specifics
	Configuration flinkConfig,
	MesosServices mesosServices,
	MesosConfiguration mesosConfig,
	MesosTaskManagerParameters taskManagerParameters,
	ContainerSpecification taskManagerContainerSpec,
	JobManagerMetricGroup jobManagerMetricGroup) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		new ClusterInformation("localhost", 1234),
		fatalErrorHandler,
		flinkConfig,
		mesosServices,
		mesosConfig,
		taskManagerParameters,
		taskManagerContainerSpec,
		null,
		jobManagerMetricGroup);
}

Example #7

Source File: MesosResourceManagerTest.java From Flink-CEPplus with Apache License 2.0

5 votes

MockMesosResourceManagerRuntimeServices() throws Exception {
	schedulerDriver = mock(SchedulerDriver.class);

	mesosConfig = mock(MesosConfiguration.class);
	when(mesosConfig.frameworkInfo()).thenReturn(Protos.FrameworkInfo.newBuilder());
	when(mesosConfig.withFrameworkInfo(any(Protos.FrameworkInfo.Builder.class))).thenReturn(mesosConfig);
	when(mesosConfig.createDriver(any(Scheduler.class), anyBoolean())).thenReturn(schedulerDriver);

	workerStore = mock(MesosWorkerStore.class);
	when(workerStore.getFrameworkID()).thenReturn(Option.<Protos.FrameworkID>empty());

	artifactServer = mock(MesosArtifactServer.class);
}

Example #8

Source File: MesosResourceManagerTest.java From Flink-CEPplus with Apache License 2.0

5 votes

public TestingMesosResourceManager(
	RpcService rpcService,
	String resourceManagerEndpointId,
	ResourceID resourceId,
	HighAvailabilityServices highAvailabilityServices,
	HeartbeatServices heartbeatServices,
	SlotManager slotManager,
	MetricRegistry metricRegistry,
	JobLeaderIdService jobLeaderIdService,
	FatalErrorHandler fatalErrorHandler,

	// Mesos specifics
	Configuration flinkConfig,
	MesosServices mesosServices,
	MesosConfiguration mesosConfig,
	MesosTaskManagerParameters taskManagerParameters,
	ContainerSpecification taskManagerContainerSpec,
	JobManagerMetricGroup jobManagerMetricGroup) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		new ClusterInformation("localhost", 1234),
		fatalErrorHandler,
		flinkConfig,
		mesosServices,
		mesosConfig,
		taskManagerParameters,
		taskManagerContainerSpec,
		null,
		jobManagerMetricGroup);
}

Example #9

Source File: LaunchableMesosWorker.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Construct a launchable Mesos worker.
 * @param resolver The resolver for retrieving artifacts (e.g. jars, configuration)
 * @param params the TM parameters such as memory, cpu to acquire.
 * @param containerSpec an abstract container specification for launch time.
 * @param taskID the taskID for this worker.
 */
public LaunchableMesosWorker(
		MesosArtifactResolver resolver,
		MesosTaskManagerParameters params,
		ContainerSpecification containerSpec,
		Protos.TaskID taskID,
		MesosConfiguration mesosConfiguration) {
	this.resolver = Preconditions.checkNotNull(resolver);
	this.containerSpec = Preconditions.checkNotNull(containerSpec);
	this.params = Preconditions.checkNotNull(params);
	this.taskID = Preconditions.checkNotNull(taskID);
	this.mesosConfiguration = Preconditions.checkNotNull(mesosConfiguration);

	this.taskRequest = new Request();
}

Example #10

Source File: MesosEntrypointUtils.java From flink with Apache License 2.0

4 votes

/**
 * Loads and validates the Mesos scheduler configuration.
 * @param flinkConfig the global configuration.
 * @param hostname the hostname to advertise to the Mesos master.
 */
public static MesosConfiguration createMesosSchedulerConfiguration(Configuration flinkConfig, String hostname) {

	Protos.FrameworkInfo.Builder frameworkInfo = Protos.FrameworkInfo.newBuilder()
		.setHostname(hostname);
	Protos.Credential.Builder credential = null;

	if (!flinkConfig.contains(MesosOptions.MASTER_URL)) {
		throw new IllegalConfigurationException(MesosOptions.MASTER_URL.key() + " must be configured.");
	}
	String masterUrl = flinkConfig.getString(MesosOptions.MASTER_URL);

	Duration failoverTimeout = FiniteDuration.apply(
		flinkConfig.getInteger(
			MesosOptions.FAILOVER_TIMEOUT_SECONDS),
			TimeUnit.SECONDS);
	frameworkInfo.setFailoverTimeout(failoverTimeout.toSeconds());

	frameworkInfo.setName(flinkConfig.getString(
		MesosOptions.RESOURCEMANAGER_FRAMEWORK_NAME));

	frameworkInfo.setRole(flinkConfig.getString(
		MesosOptions.RESOURCEMANAGER_FRAMEWORK_ROLE));

	frameworkInfo.setUser(flinkConfig.getString(
		MesosOptions.RESOURCEMANAGER_FRAMEWORK_USER));

	if (flinkConfig.contains(MesosOptions.RESOURCEMANAGER_FRAMEWORK_PRINCIPAL)) {
		frameworkInfo.setPrincipal(flinkConfig.getString(
			MesosOptions.RESOURCEMANAGER_FRAMEWORK_PRINCIPAL));

		credential = Protos.Credential.newBuilder();
		credential.setPrincipal(frameworkInfo.getPrincipal());

		// some environments use a side-channel to communicate the secret to Mesos,
		// and thus don't set the 'secret' configuration setting
		if (flinkConfig.contains(MesosOptions.RESOURCEMANAGER_FRAMEWORK_SECRET)) {
			credential.setSecret(flinkConfig.getString(
				MesosOptions.RESOURCEMANAGER_FRAMEWORK_SECRET));
		}
	}

	MesosConfiguration mesos =
		new MesosConfiguration(masterUrl, frameworkInfo, scala.Option.apply(credential));

	return mesos;
}

Example #11

Source File: MesosResourceManager.java From flink with Apache License 2.0

4 votes

public MesosResourceManager(
		// base class
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		// Mesos specifics
		Configuration flinkConfig,
		MesosServices mesosServices,
		MesosConfiguration mesosConfig,
		MesosTaskManagerParameters taskManagerParameters,
		ContainerSpecification taskManagerContainerSpec,
		@Nullable String webUiUrl,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		jobManagerMetricGroup);

	this.mesosServices = Preconditions.checkNotNull(mesosServices);
	this.actorSystem = Preconditions.checkNotNull(mesosServices.getLocalActorSystem());

	this.flinkConfig = Preconditions.checkNotNull(flinkConfig);
	this.mesosConfig = Preconditions.checkNotNull(mesosConfig);

	this.artifactServer = Preconditions.checkNotNull(mesosServices.getArtifactServer());

	this.taskManagerParameters = Preconditions.checkNotNull(taskManagerParameters);
	this.taskManagerContainerSpec = Preconditions.checkNotNull(taskManagerContainerSpec);
	this.webUiUrl = webUiUrl;

	this.workersInNew = new HashMap<>(8);
	this.workersInLaunch = new HashMap<>(8);
	this.workersBeingReturned = new HashMap<>(8);

	this.slotsPerWorker = createWorkerSlotProfiles(flinkConfig);
}

Example #12

Source File: MesosEntrypointUtils.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Loads and validates the Mesos scheduler configuration.
 * @param flinkConfig the global configuration.
 * @param hostname the hostname to advertise to the Mesos master.
 */
public static MesosConfiguration createMesosSchedulerConfiguration(Configuration flinkConfig, String hostname) {

	Protos.FrameworkInfo.Builder frameworkInfo = Protos.FrameworkInfo.newBuilder()
		.setHostname(hostname);
	Protos.Credential.Builder credential = null;

	if (!flinkConfig.contains(MesosOptions.MASTER_URL)) {
		throw new IllegalConfigurationException(MesosOptions.MASTER_URL.key() + " must be configured.");
	}
	String masterUrl = flinkConfig.getString(MesosOptions.MASTER_URL);

	Duration failoverTimeout = FiniteDuration.apply(
		flinkConfig.getInteger(
			MesosOptions.FAILOVER_TIMEOUT_SECONDS),
			TimeUnit.SECONDS);
	frameworkInfo.setFailoverTimeout(failoverTimeout.toSeconds());

	frameworkInfo.setName(flinkConfig.getString(
		MesosOptions.RESOURCEMANAGER_FRAMEWORK_NAME));

	frameworkInfo.setRole(flinkConfig.getString(
		MesosOptions.RESOURCEMANAGER_FRAMEWORK_ROLE));

	frameworkInfo.setUser(flinkConfig.getString(
		MesosOptions.RESOURCEMANAGER_FRAMEWORK_USER));

	if (flinkConfig.contains(MesosOptions.RESOURCEMANAGER_FRAMEWORK_PRINCIPAL)) {
		frameworkInfo.setPrincipal(flinkConfig.getString(
			MesosOptions.RESOURCEMANAGER_FRAMEWORK_PRINCIPAL));

		credential = Protos.Credential.newBuilder();
		credential.setPrincipal(frameworkInfo.getPrincipal());

		// some environments use a side-channel to communicate the secret to Mesos,
		// and thus don't set the 'secret' configuration setting
		if (flinkConfig.contains(MesosOptions.RESOURCEMANAGER_FRAMEWORK_SECRET)) {
			credential.setSecret(flinkConfig.getString(
				MesosOptions.RESOURCEMANAGER_FRAMEWORK_SECRET));
		}
	}

	MesosConfiguration mesos =
		new MesosConfiguration(masterUrl, frameworkInfo, scala.Option.apply(credential));

	return mesos;
}

Example #13

Source File: MesosResourceManagerFactory.java From flink with Apache License 2.0

4 votes

public MesosResourceManagerFactory(@Nonnull MesosServices mesosServices, @Nonnull MesosConfiguration schedulerConfiguration, @Nonnull MesosTaskManagerParameters taskManagerParameters, @Nonnull ContainerSpecification taskManagerContainerSpec) {
	this.mesosServices = mesosServices;
	this.schedulerConfiguration = schedulerConfiguration;
	this.taskManagerParameters = taskManagerParameters;
	this.taskManagerContainerSpec = taskManagerContainerSpec;
}

Example #14

Source File: MesosResourceManager.java From flink with Apache License 2.0

4 votes

public MesosResourceManager(
		// base class
		RpcService rpcService,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		ResourceManagerPartitionTrackerFactory clusterPartitionTrackerFactory,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		// Mesos specifics
		Configuration flinkConfig,
		MesosServices mesosServices,
		MesosConfiguration mesosConfig,
		MesosTaskManagerParameters taskManagerParameters,
		ContainerSpecification taskManagerContainerSpec,
		@Nullable String webUiUrl,
		ResourceManagerMetricGroup resourceManagerMetricGroup) {
	super(
		rpcService,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		clusterPartitionTrackerFactory,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		resourceManagerMetricGroup,
		AkkaUtils.getTimeoutAsTime(flinkConfig));

	this.mesosServices = Preconditions.checkNotNull(mesosServices);
	this.actorSystem = Preconditions.checkNotNull(mesosServices.getLocalActorSystem());

	this.flinkConfig = Preconditions.checkNotNull(flinkConfig);
	this.mesosConfig = Preconditions.checkNotNull(mesosConfig);

	this.artifactServer = Preconditions.checkNotNull(mesosServices.getArtifactServer());

	this.taskManagerParameters = Preconditions.checkNotNull(taskManagerParameters);
	this.taskManagerContainerSpec = Preconditions.checkNotNull(taskManagerContainerSpec);
	this.webUiUrl = webUiUrl;

	this.workersInNew = new HashMap<>(8);
	this.workersInLaunch = new HashMap<>(8);
	this.workersBeingReturned = new HashMap<>(8);
}

Example #15

Source File: MesosResourceManagerFactory.java From Flink-CEPplus with Apache License 2.0

4 votes

public MesosResourceManagerFactory(@Nonnull MesosServices mesosServices, @Nonnull MesosConfiguration schedulerConfiguration, @Nonnull MesosTaskManagerParameters taskManagerParameters, @Nonnull ContainerSpecification taskManagerContainerSpec) {
	this.mesosServices = mesosServices;
	this.schedulerConfiguration = schedulerConfiguration;
	this.taskManagerParameters = taskManagerParameters;
	this.taskManagerContainerSpec = taskManagerContainerSpec;
}

Example #16

Source File: MesosResourceManagerFactory.java From flink with Apache License 2.0

4 votes

public MesosResourceManagerFactory(@Nonnull MesosServices mesosServices, @Nonnull MesosConfiguration schedulerConfiguration) {
	this.mesosServices = mesosServices;
	this.schedulerConfiguration = schedulerConfiguration;
}

Example #17

Source File: MesosResourceManager.java From Flink-CEPplus with Apache License 2.0

4 votes

public MesosResourceManager(
		// base class
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		// Mesos specifics
		Configuration flinkConfig,
		MesosServices mesosServices,
		MesosConfiguration mesosConfig,
		MesosTaskManagerParameters taskManagerParameters,
		ContainerSpecification taskManagerContainerSpec,
		@Nullable String webUiUrl,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		jobManagerMetricGroup);

	this.mesosServices = Preconditions.checkNotNull(mesosServices);
	this.actorSystem = Preconditions.checkNotNull(mesosServices.getLocalActorSystem());

	this.flinkConfig = Preconditions.checkNotNull(flinkConfig);
	this.mesosConfig = Preconditions.checkNotNull(mesosConfig);

	this.artifactServer = Preconditions.checkNotNull(mesosServices.getArtifactServer());

	this.taskManagerParameters = Preconditions.checkNotNull(taskManagerParameters);
	this.taskManagerContainerSpec = Preconditions.checkNotNull(taskManagerContainerSpec);
	this.webUiUrl = webUiUrl;

	this.workersInNew = new HashMap<>(8);
	this.workersInLaunch = new HashMap<>(8);
	this.workersBeingReturned = new HashMap<>(8);

	final ContaineredTaskManagerParameters containeredTaskManagerParameters = taskManagerParameters.containeredParameters();
	this.slotsPerWorker = createSlotsPerWorker(containeredTaskManagerParameters.numSlots());
}