Java Code Examples for org.apache.mesos.Protos#Volume

The following examples show how to use org.apache.mesos.Protos#Volume . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MesosTaskManagerParametersTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testBuildVolumes() throws Exception {
	List<Protos.Volume> vols;
	assertEquals(MesosTaskManagerParameters.buildVolumes(Option.<String>apply(null)).size(), 0);
	String spec1 = "/host/path:/container/path:RO,/container/path:ro,/host/path:/container/path,/container/path";
	vols = MesosTaskManagerParameters.buildVolumes(Option.<String>apply(spec1));
	assertEquals(vols.size(), 4);
	assertEquals("/container/path", vols.get(0).getContainerPath());
	assertEquals("/host/path", vols.get(0).getHostPath());
	assertEquals(Protos.Volume.Mode.RO, vols.get(0).getMode());
	assertEquals("/container/path", vols.get(1).getContainerPath());
	assertEquals(Protos.Volume.Mode.RO, vols.get(1).getMode());
	assertEquals("/container/path", vols.get(2).getContainerPath());
	assertEquals("/host/path", vols.get(2).getHostPath());
	assertEquals(Protos.Volume.Mode.RW, vols.get(2).getMode());
	assertEquals("/container/path", vols.get(3).getContainerPath());
	assertEquals(Protos.Volume.Mode.RW, vols.get(3).getMode());

	// should handle empty strings, but not error
	assertEquals(0, MesosTaskManagerParameters.buildVolumes(Option.<String>apply("")).size());
}
 
Example 2
Source File: ResourceBuilderTest.java    From dcos-commons with Apache License 2.0 6 votes vote down vote up
private static void validateDisk(Protos.Resource resource, Optional<String> resourceId, Optional<String> namespace, Optional<String> frameworkId) {
    Assert.assertTrue(resource.hasDisk());

    Protos.Resource.DiskInfo diskInfo = resource.getDisk();
    Assert.assertTrue(diskInfo.hasPersistence());

    Protos.Resource.DiskInfo.Persistence persistence = diskInfo.getPersistence();
    Assert.assertEquals(36, persistence.getId().length());
    Assert.assertEquals(TestConstants.PRINCIPAL, persistence.getPrincipal());

    Assert.assertTrue(diskInfo.hasVolume());
    Protos.Volume volume = diskInfo.getVolume();
    Assert.assertEquals(TestConstants.CONTAINER_PATH, volume.getContainerPath());
    Assert.assertEquals(Protos.Volume.Mode.RW, volume.getMode());

    validateScalarResource(resource, resourceId, namespace, frameworkId);
}
 
Example 3
Source File: MesosTaskManagerParametersTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testBuildVolumes() throws Exception {
	List<Protos.Volume> vols;
	assertEquals(MesosTaskManagerParameters.buildVolumes(Option.<String>apply(null)).size(), 0);
	String spec1 = "/host/path:/container/path:RO,/container/path:ro,/host/path:/container/path,/container/path";
	vols = MesosTaskManagerParameters.buildVolumes(Option.<String>apply(spec1));
	assertEquals(vols.size(), 4);
	assertEquals("/container/path", vols.get(0).getContainerPath());
	assertEquals("/host/path", vols.get(0).getHostPath());
	assertEquals(Protos.Volume.Mode.RO, vols.get(0).getMode());
	assertEquals("/container/path", vols.get(1).getContainerPath());
	assertEquals(Protos.Volume.Mode.RO, vols.get(1).getMode());
	assertEquals("/container/path", vols.get(2).getContainerPath());
	assertEquals("/host/path", vols.get(2).getHostPath());
	assertEquals(Protos.Volume.Mode.RW, vols.get(2).getMode());
	assertEquals("/container/path", vols.get(3).getContainerPath());
	assertEquals(Protos.Volume.Mode.RW, vols.get(3).getMode());

	// should handle empty strings, but not error
	assertEquals(0, MesosTaskManagerParameters.buildVolumes(Option.<String>apply("")).size());
}
 
Example 4
Source File: PodInfoBuilder.java    From dcos-commons with Apache License 2.0 6 votes vote down vote up
private static Collection<Protos.Volume> getExecutorInfoHostVolumes(Collection<HostVolumeSpec> hostVolumeSpecs) {
  Collection<Protos.Volume> volumes = new ArrayList<>();

  for (HostVolumeSpec hostVolumeSpec : hostVolumeSpecs) {
    if (hostVolumeSpec.getMode().isPresent()) {
      //set mode on host volume if defined or revert to RW as default
      volumes.add(Protos.Volume.newBuilder()
          .setHostPath(hostVolumeSpec.getHostPath())
          .setContainerPath(hostVolumeSpec.getContainerPath())
          .setMode(hostVolumeSpec.getMode().get())
          .build());

    } else {
      volumes.add(Protos.Volume.newBuilder()
          .setHostPath(hostVolumeSpec.getHostPath())
          .setContainerPath(hostVolumeSpec.getContainerPath())
          .setMode(Protos.Volume.Mode.RW)
          .build());
    }
  }

  return volumes;
}
 
Example 5
Source File: PodInfoBuilder.java    From dcos-commons with Apache License 2.0 6 votes vote down vote up
private static Collection<Protos.Volume> getExecutorInfoSecretVolumes(Collection<SecretSpec> secretSpecs) {
  Collection<Protos.Volume> volumes = new ArrayList<>();

  for (SecretSpec secretSpec : secretSpecs) {
    if (secretSpec.getFilePath().isPresent()) {
      volumes.add(Protos.Volume.newBuilder()
          .setSource(Protos.Volume.Source.newBuilder()
              .setType(Protos.Volume.Source.Type.SECRET)
              .setSecret(getReferenceSecret(secretSpec.getSecretPath()))
              .build())
          .setContainerPath(secretSpec.getFilePath().get())
          .setMode(Protos.Volume.Mode.RO)
          .build());
    }
  }
  return volumes;
}
 
Example 6
Source File: MesosTaskManagerParameters.java    From flink with Apache License 2.0 5 votes vote down vote up
public MesosTaskManagerParameters(
		int gpus,
		int disk,
		int network,
		ContainerType containerType,
		Option<String> containerImageName,
		ContaineredTaskManagerParameters containeredParameters,
		List<Protos.Volume> containerVolumes,
		List<Protos.Parameter> dockerParameters,
		boolean dockerForcePullImage,
		List<ConstraintEvaluator> constraints,
		String command,
		Option<String> bootstrapCommand,
		Option<String> taskManagerHostname,
		List<String> uris) {

	this.gpus = gpus;
	this.disk = disk;
	this.network = network;
	this.containerType = Preconditions.checkNotNull(containerType);
	this.containerImageName = Preconditions.checkNotNull(containerImageName);
	this.containeredParameters = Preconditions.checkNotNull(containeredParameters);
	this.containerVolumes = Preconditions.checkNotNull(containerVolumes);
	this.dockerParameters = Preconditions.checkNotNull(dockerParameters);
	this.dockerForcePullImage = dockerForcePullImage;
	this.constraints = Preconditions.checkNotNull(constraints);
	this.command = Preconditions.checkNotNull(command);
	this.bootstrapCommand = Preconditions.checkNotNull(bootstrapCommand);
	this.taskManagerHostname = Preconditions.checkNotNull(taskManagerHostname);
	this.uris = Preconditions.checkNotNull(uris);
}
 
Example 7
Source File: TLSEvaluationStageTest.java    From dcos-commons with Apache License 2.0 5 votes vote down vote up
private static Optional<String> findSecretStorePath(Protos.ContainerInfo container, String path) {
    Optional<Protos.Volume> volume = container.getVolumesList().stream()
            .filter(v -> v.getContainerPath().equals(path))
            .findAny();
    return volume.isPresent()
            ? Optional.ofNullable(volume.get().getSource().getSecret().getReference().getName())
            : Optional.empty();
}
 
Example 8
Source File: MesosTaskManagerParameters.java    From flink with Apache License 2.0 5 votes vote down vote up
public MesosTaskManagerParameters(
		double cpus,
		int gpus,
		int disk,
		ContainerType containerType,
		Option<String> containerImageName,
		ContaineredTaskManagerParameters containeredParameters,
		List<Protos.Volume> containerVolumes,
		List<Protos.Parameter> dockerParameters,
		boolean dockerForcePullImage,
		List<ConstraintEvaluator> constraints,
		String command,
		Option<String> bootstrapCommand,
		Option<String> taskManagerHostname,
		List<String> uris) {

	this.cpus = cpus;
	this.gpus = gpus;
	this.disk = disk;
	this.containerType = Preconditions.checkNotNull(containerType);
	this.containerImageName = Preconditions.checkNotNull(containerImageName);
	this.containeredParameters = Preconditions.checkNotNull(containeredParameters);
	this.containerVolumes = Preconditions.checkNotNull(containerVolumes);
	this.dockerParameters = Preconditions.checkNotNull(dockerParameters);
	this.dockerForcePullImage = dockerForcePullImage;
	this.constraints = Preconditions.checkNotNull(constraints);
	this.command = Preconditions.checkNotNull(command);
	this.bootstrapCommand = Preconditions.checkNotNull(bootstrapCommand);
	this.taskManagerHostname = Preconditions.checkNotNull(taskManagerHostname);
	this.uris = Preconditions.checkNotNull(uris);
}
 
Example 9
Source File: PodInfoBuilder.java    From dcos-commons with Apache License 2.0 5 votes vote down vote up
public void setExecutorVolume(VolumeSpec volumeSpec) {
  // Volumes on the executor must be declared in each TaskInfo.ContainerInfo to be shared among them.

  Protos.Volume executorVolume = getVolume(volumeSpec);

  for (Protos.TaskInfo.Builder t : getTaskBuilders()) {
    t.getContainerBuilder()
        .setType(Protos.ContainerInfo.Type.MESOS)
        .addVolumes(executorVolume);
  }

}
 
Example 10
Source File: TLSEvaluationStage.java    From dcos-commons with Apache License 2.0 5 votes vote down vote up
private static Protos.Volume getSecretVolume(TLSArtifactPaths.Entry entry) {
  Protos.Volume.Builder volumeBuilder = Protos.Volume.newBuilder()
      .setContainerPath(entry.mountPath)
      .setMode(Protos.Volume.Mode.RO);
  Protos.Volume.Source.Builder sourceBuilder = volumeBuilder.getSourceBuilder()
      .setType(Protos.Volume.Source.Type.SECRET);
  sourceBuilder.getSecretBuilder()
      .setType(Protos.Secret.Type.REFERENCE)
      .getReferenceBuilder().setName(entry.secretStorePath);
  return volumeBuilder.build();
}
 
Example 11
Source File: MesosTaskManagerParameters.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public MesosTaskManagerParameters(
		double cpus,
		int gpus,
		ContainerType containerType,
		Option<String> containerImageName,
		ContaineredTaskManagerParameters containeredParameters,
		List<Protos.Volume> containerVolumes,
		List<Protos.Parameter> dockerParameters,
		boolean dockerForcePullImage,
		List<ConstraintEvaluator> constraints,
		String command,
		Option<String> bootstrapCommand,
		Option<String> taskManagerHostname,
		List<String> uris) {

	this.cpus = cpus;
	this.gpus = gpus;
	this.containerType = Preconditions.checkNotNull(containerType);
	this.containerImageName = Preconditions.checkNotNull(containerImageName);
	this.containeredParameters = Preconditions.checkNotNull(containeredParameters);
	this.containerVolumes = Preconditions.checkNotNull(containerVolumes);
	this.dockerParameters = Preconditions.checkNotNull(dockerParameters);
	this.dockerForcePullImage = dockerForcePullImage;
	this.constraints = Preconditions.checkNotNull(constraints);
	this.command = Preconditions.checkNotNull(command);
	this.bootstrapCommand = Preconditions.checkNotNull(bootstrapCommand);
	this.taskManagerHostname = Preconditions.checkNotNull(taskManagerHostname);
	this.uris = Preconditions.checkNotNull(uris);
}
 
Example 12
Source File: TLSEvaluationStage.java    From dcos-commons with Apache License 2.0 5 votes vote down vote up
private static Set<Protos.Volume> getExecutorInfoSecretVolumes(
    TransportEncryptionSpec spec, TLSArtifactPaths tlsArtifactPaths)
{

  Collection<TLSArtifactPaths.Entry> paths =
      tlsArtifactPaths.getPathsForType(spec.getType(), spec.getName());
  return paths.stream()
      .map(TLSEvaluationStage::getSecretVolume)
      .collect(Collectors.toSet());
}
 
Example 13
Source File: PodInfoBuilder.java    From dcos-commons with Apache License 2.0 4 votes vote down vote up
/**
 * Get the ContainerInfo for either an Executor or a Task. Since we support both default and custom executors at
 * the moment, there is some conditional logic in here -- with the default executor, things like rlimits and images
 * must be specified at the task level only, while secrets volumes must be specified at the executor level.
 *
 * @param podSpec            The Spec for the task or executor that this container is being attached to
 * @param addExtraParameters Add rlimits and docker image (if task), or secrets volumes if executor
 * @param isTaskContainer    Whether this container is being attached to a TaskInfo rather than ExecutorInfo
 * @return the ContainerInfo to be attached
 */
private Protos.ContainerInfo getContainerInfo(
    PodSpec podSpec, boolean addExtraParameters, boolean isTaskContainer)
{
  Collection<Protos.Volume> secretVolumes = getExecutorInfoSecretVolumes(podSpec.getSecrets());
  Collection<Protos.Volume> hostVolumes = getExecutorInfoHostVolumes(podSpec.getHostVolumes());
  Protos.ContainerInfo.Builder containerInfo = Protos.ContainerInfo.newBuilder()
      .setType(Protos.ContainerInfo.Type.MESOS);

  if (isTaskContainer) {
    containerInfo.getLinuxInfoBuilder().setSharePidNamespace(podSpec.getSharePidNamespace());
    // Isolate the tmp directory of tasks
    // switch to SANDBOX SELF after dc/os 1.13

    containerInfo.addVolumes(Protos.Volume.newBuilder()
            .setContainerPath("/tmp")
            .setHostPath("tmp")
            .setMode(Protos.Volume.Mode.RW));

    LOGGER.info("Setting seccomp info unconfined: {} profile: {}",
            podSpec.getSeccompUnconfined(),
            podSpec.getSeccompProfileName());

    if (podSpec.getSeccompUnconfined() != null && podSpec.getSeccompUnconfined()) {
      containerInfo.getLinuxInfoBuilder().setSeccomp(Protos.SeccompInfo.newBuilder()
              .setUnconfined(podSpec.getSeccompUnconfined())
              .build());
    }

    if (podSpec.getSeccompProfileName().isPresent()) {
      containerInfo.getLinuxInfoBuilder().setSeccomp(Protos.SeccompInfo.newBuilder()
              .setProfileName(podSpec.getSeccompProfileName().get())
              .build());
    }
  } else {

    if (podSpec.getSharedMemory().isPresent()) {
      containerInfo.getLinuxInfoBuilder().setIpcMode(podSpec.getSharedMemory().get());
    }

    if (podSpec.getSharedMemorySize().isPresent()) {
      containerInfo.getLinuxInfoBuilder().setShmSize(podSpec.getSharedMemorySize().get());
    }
  }

  for (Protos.Volume hostVolume : hostVolumes) {
    containerInfo.addVolumes(hostVolume);
  }

  if (!podSpec.getImage().isPresent()
      && podSpec.getNetworks().isEmpty()
      && podSpec.getRLimits().isEmpty()
      && secretVolumes.isEmpty())
  {
    // Nothing left to do.
    return containerInfo.build();
  }

  boolean shouldAddImage =
      podSpec.getImage().isPresent() &&
          addExtraParameters &&
          isTaskContainer;

  if (shouldAddImage) {
    containerInfo.getMesosBuilder().getImageBuilder()
        .setType(Protos.Image.Type.DOCKER)
        .getDockerBuilder().setName(podSpec.getImage().get());
  }

  // With the default executor, all NetworkInfos must be defined on the executor itself rather than individual
  // tasks. This check can be made much less ugly once the custom executor no longer need be supported.
  if (!podSpec.getNetworks().isEmpty() && !isTaskContainer) {
    LOGGER.info("Adding NetworkInfos for networks: {}",
        podSpec.getNetworks().stream().map(n -> n.getName()).collect(Collectors.toList()));
    containerInfo.addAllNetworkInfos(
        podSpec.getNetworks().stream().map(PodInfoBuilder::getNetworkInfo).collect(Collectors.toList()));
  }

  if (!podSpec.getRLimits().isEmpty() && addExtraParameters) {
    containerInfo.setRlimitInfo(getRLimitInfo(podSpec.getRLimits()));
  }

  if (addExtraParameters) {
    for (Protos.Volume secretVolume : secretVolumes) {
      containerInfo.addVolumes(secretVolume);
    }
  }

  return containerInfo.build();
}
 
Example 14
Source File: MesosTaskManagerParameters.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Get the container volumes string.
 */
public List<Protos.Volume> containerVolumes() {
	return containerVolumes;
}
 
Example 15
Source File: TLSEvaluationStage.java    From dcos-commons with Apache License 2.0 4 votes vote down vote up
@Override
public EvaluationOutcome evaluate(MesosResourcePool mesosResourcePool,
                                  PodInfoBuilder podInfoBuilder)
{
  TaskSpec taskSpec = podInfoBuilder.getPodInstance().getPod().getTasks().stream()
      .filter(task -> task.getName().equals(taskName))
      .findFirst()
      .get();
  if (taskSpec.getTransportEncryption().isEmpty()) {
    return EvaluationOutcome.pass(
        this,
        "No TLS specs found for task").build();
  }

  CertificateNamesGenerator certificateNamesGenerator =
      new CertificateNamesGenerator(
          serviceName, taskSpec,
          podInfoBuilder.getPodInstance(),
          schedulerConfig);
  TLSArtifactPaths tlsArtifactPaths = new TLSArtifactPaths(
      namespace,
      CommonIdUtils.getTaskInstanceName(podInfoBuilder.getPodInstance(), taskName),
      certificateNamesGenerator.getSANsHash());

  Collection<TransportEncryptionSpec> transportEncryptionSpecs =
      taskSpec.getTransportEncryption();
  logger.info("Processing TLS info for {} elements of {}",
      transportEncryptionSpecs.size(),
      transportEncryptionSpecs);
  for (TransportEncryptionSpec transportEncryptionSpec : transportEncryptionSpecs) {
    try {
      tlsArtifactsUpdater.update(
          tlsArtifactPaths, certificateNamesGenerator, transportEncryptionSpec.getName());
    } catch (Exception e) {
      logger.error(String.format("Failed to process certificates for %s", taskName), e);
      return EvaluationOutcome.fail(
          this,
          "Failed to store TLS artifacts for task %s because of exception: %s",
          taskName,
          e).build();
    }

    Set<Protos.Volume> existingVolumes = podInfoBuilder.getTaskBuilder(taskName)
        .getContainerBuilder()
        .getVolumesList()
        .stream()
        .collect(Collectors.toSet());
    logger.debug("Existing volumes for {}: {}",
        taskName,
        existingVolumes.stream().map(v -> v.getContainerPath()).toArray());

    Set<Protos.Volume> additionalVolumes = getExecutorInfoSecretVolumes(
        transportEncryptionSpec,
        tlsArtifactPaths
    );
    logger.debug("Required volumes for {}: {}",
        taskName,
        additionalVolumes.stream().map(v -> v.getContainerPath()).toArray());

    if (additionalVolumes.removeAll(existingVolumes)) {
      logger.debug("Duplicate volumes for {} removed. Remaining: {}",
          taskName,
          additionalVolumes.stream().map(v -> v.getContainerPath()).toArray());
    }

    // Share keys to the task container
    podInfoBuilder
        .getTaskBuilder(taskName)
        .getContainerBuilder()
        .addAllVolumes(additionalVolumes);
  }

  return EvaluationOutcome.pass(
      this,
      "TLS certificate created and added to the task").build();
}
 
Example 16
Source File: MesosTaskManagerParameters.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Create the Mesos TaskManager parameters.
 *
 * @param flinkConfig the TM configuration.
 */
public static MesosTaskManagerParameters create(Configuration flinkConfig) {

	List<ConstraintEvaluator> constraints = parseConstraints(flinkConfig.getString(MESOS_CONSTRAINTS_HARD_HOSTATTR));

	// parse the common parameters
	ContaineredTaskManagerParameters containeredParameters = createContaineredTaskManagerParameters(flinkConfig);

	int gpus = flinkConfig.getInteger(MESOS_RM_TASKS_GPUS);

	if (gpus < 0) {
		throw new IllegalConfigurationException(MESOS_RM_TASKS_GPUS.key() +
			" cannot be negative");
	}

	int disk = flinkConfig.getInteger(MESOS_RM_TASKS_DISK_MB);

	int network = flinkConfig.getInteger(MESOS_RM_TASKS_NETWORK_MB_PER_SEC);

	// parse the containerization parameters
	String imageName = flinkConfig.getString(MESOS_RM_CONTAINER_IMAGE_NAME);

	ContainerType containerType;
	String containerTypeString = flinkConfig.getString(MESOS_RM_CONTAINER_TYPE);
	switch (containerTypeString) {
		case MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE_MESOS:
			containerType = ContainerType.MESOS;
			break;
		case MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE_DOCKER:
			containerType = ContainerType.DOCKER;
			if (imageName == null || imageName.length() == 0) {
				throw new IllegalConfigurationException(MESOS_RM_CONTAINER_IMAGE_NAME.key() +
					" must be specified for docker container type");
			}
			break;
		default:
			throw new IllegalConfigurationException("invalid container type: " + containerTypeString);
	}

	Option<String> containerVolOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_VOLUMES));

	Option<String> dockerParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_DOCKER_PARAMETERS));

	Option<String> uriParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_TM_URIS));

	boolean dockerForcePullImage = flinkConfig.getBoolean(MESOS_RM_CONTAINER_DOCKER_FORCE_PULL_IMAGE);

	List<Protos.Volume> containerVolumes = buildVolumes(containerVolOpt);

	List<Protos.Parameter> dockerParameters = buildDockerParameters(dockerParamsOpt);

	List<String> uris = buildUris(uriParamsOpt);

	//obtain Task Manager Host Name from the configuration
	Option<String> taskManagerHostname = Option.apply(flinkConfig.getString(MESOS_TM_HOSTNAME));

	//obtain command-line from the configuration
	String tmCommand = flinkConfig.getString(MESOS_TM_CMD);
	Option<String> tmBootstrapCommand = Option.apply(flinkConfig.getString(MESOS_TM_BOOTSTRAP_CMD));

	return new MesosTaskManagerParameters(
		gpus,
		disk,
		network,
		containerType,
		Option.apply(imageName),
		containeredParameters,
		containerVolumes,
		dockerParameters,
		dockerForcePullImage,
		constraints,
		tmCommand,
		tmBootstrapCommand,
		taskManagerHostname,
		uris);
}
 
Example 17
Source File: MesosTaskManagerParameters.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Create the Mesos TaskManager parameters.
 *
 * @param flinkConfig the TM configuration.
 */
public static MesosTaskManagerParameters create(Configuration flinkConfig) {

	List<ConstraintEvaluator> constraints = parseConstraints(flinkConfig.getString(MESOS_CONSTRAINTS_HARD_HOSTATTR));
	// parse the common parameters
	ContaineredTaskManagerParameters containeredParameters = ContaineredTaskManagerParameters.create(
		flinkConfig,
		flinkConfig.getInteger(MESOS_RM_TASKS_MEMORY_MB),
		flinkConfig.getInteger(MESOS_RM_TASKS_SLOTS));

	double cpus = flinkConfig.getDouble(MESOS_RM_TASKS_CPUS);
	if (cpus <= 0.0) {
		cpus = Math.max(containeredParameters.numSlots(), 1.0);
	}

	int gpus = flinkConfig.getInteger(MESOS_RM_TASKS_GPUS);

	if (gpus < 0) {
		throw new IllegalConfigurationException(MESOS_RM_TASKS_GPUS.key() +
			" cannot be negative");
	}

	int disk = flinkConfig.getInteger(MESOS_RM_TASKS_DISK_MB);

	// parse the containerization parameters
	String imageName = flinkConfig.getString(MESOS_RM_CONTAINER_IMAGE_NAME);

	ContainerType containerType;
	String containerTypeString = flinkConfig.getString(MESOS_RM_CONTAINER_TYPE);
	switch (containerTypeString) {
		case MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE_MESOS:
			containerType = ContainerType.MESOS;
			break;
		case MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE_DOCKER:
			containerType = ContainerType.DOCKER;
			if (imageName == null || imageName.length() == 0) {
				throw new IllegalConfigurationException(MESOS_RM_CONTAINER_IMAGE_NAME.key() +
					" must be specified for docker container type");
			}
			break;
		default:
			throw new IllegalConfigurationException("invalid container type: " + containerTypeString);
	}

	Option<String> containerVolOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_VOLUMES));

	Option<String> dockerParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_DOCKER_PARAMETERS));

	Option<String> uriParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_TM_URIS));

	boolean dockerForcePullImage = flinkConfig.getBoolean(MESOS_RM_CONTAINER_DOCKER_FORCE_PULL_IMAGE);

	List<Protos.Volume> containerVolumes = buildVolumes(containerVolOpt);

	List<Protos.Parameter> dockerParameters = buildDockerParameters(dockerParamsOpt);

	List<String> uris = buildUris(uriParamsOpt);

	//obtain Task Manager Host Name from the configuration
	Option<String> taskManagerHostname = Option.apply(flinkConfig.getString(MESOS_TM_HOSTNAME));

	//obtain command-line from the configuration
	String tmCommand = flinkConfig.getString(MESOS_TM_CMD);
	Option<String> tmBootstrapCommand = Option.apply(flinkConfig.getString(MESOS_TM_BOOTSTRAP_CMD));

	return new MesosTaskManagerParameters(
		cpus,
		gpus,
		disk,
		containerType,
		Option.apply(imageName),
		containeredParameters,
		containerVolumes,
		dockerParameters,
		dockerForcePullImage,
		constraints,
		tmCommand,
		tmBootstrapCommand,
		taskManagerHostname,
		uris);
}
 
Example 18
Source File: EnvironmentContext.java    From Singularity with Apache License 2.0 4 votes vote down vote up
public List<Protos.Volume> getContainerVolumes() {
  return taskInfo.getContainer().getVolumesList();
}
 
Example 19
Source File: MesosTaskManagerParameters.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Create the Mesos TaskManager parameters.
 *
 * @param flinkConfig the TM configuration.
 */
public static MesosTaskManagerParameters create(Configuration flinkConfig) {

	List<ConstraintEvaluator> constraints = parseConstraints(flinkConfig.getString(MESOS_CONSTRAINTS_HARD_HOSTATTR));
	// parse the common parameters
	ContaineredTaskManagerParameters containeredParameters = ContaineredTaskManagerParameters.create(
		flinkConfig,
		flinkConfig.getInteger(MESOS_RM_TASKS_MEMORY_MB),
		flinkConfig.getInteger(MESOS_RM_TASKS_SLOTS));

	double cpus = flinkConfig.getDouble(MESOS_RM_TASKS_CPUS);
	if (cpus <= 0.0) {
		cpus = Math.max(containeredParameters.numSlots(), 1.0);
	}

	int gpus = flinkConfig.getInteger(MESOS_RM_TASKS_GPUS);

	if (gpus < 0) {
		throw new IllegalConfigurationException(MESOS_RM_TASKS_GPUS.key() +
			" cannot be negative");
	}

	// parse the containerization parameters
	String imageName = flinkConfig.getString(MESOS_RM_CONTAINER_IMAGE_NAME);

	ContainerType containerType;
	String containerTypeString = flinkConfig.getString(MESOS_RM_CONTAINER_TYPE);
	switch (containerTypeString) {
		case MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE_MESOS:
			containerType = ContainerType.MESOS;
			break;
		case MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE_DOCKER:
			containerType = ContainerType.DOCKER;
			if (imageName == null || imageName.length() == 0) {
				throw new IllegalConfigurationException(MESOS_RM_CONTAINER_IMAGE_NAME.key() +
					" must be specified for docker container type");
			}
			break;
		default:
			throw new IllegalConfigurationException("invalid container type: " + containerTypeString);
	}

	Option<String> containerVolOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_VOLUMES));

	Option<String> dockerParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_RM_CONTAINER_DOCKER_PARAMETERS));

	Option<String> uriParamsOpt = Option.<String>apply(flinkConfig.getString(MESOS_TM_URIS));

	boolean dockerForcePullImage = flinkConfig.getBoolean(MESOS_RM_CONTAINER_DOCKER_FORCE_PULL_IMAGE);

	List<Protos.Volume> containerVolumes = buildVolumes(containerVolOpt);

	List<Protos.Parameter> dockerParameters = buildDockerParameters(dockerParamsOpt);

	List<String> uris = buildUris(uriParamsOpt);

	//obtain Task Manager Host Name from the configuration
	Option<String> taskManagerHostname = Option.apply(flinkConfig.getString(MESOS_TM_HOSTNAME));

	//obtain command-line from the configuration
	String tmCommand = flinkConfig.getString(MESOS_TM_CMD);
	Option<String> tmBootstrapCommand = Option.apply(flinkConfig.getString(MESOS_TM_BOOTSTRAP_CMD));

	return new MesosTaskManagerParameters(
		cpus,
		gpus,
		containerType,
		Option.apply(imageName),
		containeredParameters,
		containerVolumes,
		dockerParameters,
		dockerForcePullImage,
		constraints,
		tmCommand,
		tmBootstrapCommand,
		taskManagerHostname,
		uris);
}
 
Example 20
Source File: MesosTaskManagerParameters.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Get the container volumes string.
 */
public List<Protos.Volume> containerVolumes() {
	return containerVolumes;
}