org.apache.mesos.Protos.CommandInfo Java Examples

The following examples show how to use org.apache.mesos.Protos.CommandInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: LaunchTaskTest.java From storm with Apache License 2.0

6 votes

/**
 * Setup testing target & sample data.
 */
public LaunchTaskTest()
{
  SlaveID slaveID = SlaveID.newBuilder().setValue("s1").build();
  this.sampleTaskInfo =
      TaskInfo.newBuilder()
              .setName("t1").setSlaveId(slaveID)
              .setTaskId(TaskID.newBuilder().setValue("id2"))
              .setExecutor(
                  ExecutorInfo.newBuilder()
                              .setExecutorId(ExecutorID.newBuilder().setValue("e1"))
                              .setCommand(CommandInfo.getDefaultInstance()))
              .build();
  this.sampleOffer =
      Offer.newBuilder()
           .setHostname("h1").setSlaveId(slaveID)
           .setId(OfferID.newBuilder().setValue("id1"))
           .setFrameworkId(FrameworkID.newBuilder().setValue("f1").build())
           .build();
  this.target = new LaunchTask(sampleTaskInfo, sampleOffer);
}

Example #2

Source File: NMExecutorCommandLineGenerator.java From incubator-myriad with Apache License 2.0

5 votes

@Override
CommandInfo generateCommandLine(ServiceResourceProfile profile,
                                ServiceConfiguration serviceConfiguration, Collection<Long> ports) {
  CommandInfo.Builder builder = CommandInfo.newBuilder();
  builder.mergeFrom(staticCommandInfo);
  builder.setEnvironment(generateEnvironment(profile, ports));
  builder.setUser(getUser());
  return builder.build();
}

Example #3

Source File: NMExecutorCommandLineGenerator.java From incubator-myriad with Apache License 2.0

5 votes

protected void generateStaticCommandLine() {
  CommandInfo.Builder builder = CommandInfo.newBuilder();
  StringBuilder cmdLine = new StringBuilder();
  appendCgroupsCmds(cmdLine);
  appendDistroExtractionCommands(cmdLine);
  appendUserSudo(cmdLine);
  cmdLine.append(YARN_NM_CMD);
  builder.setValue(String.format(CMD_FORMAT, cmdLine.toString()));
  builder.addAllUris(getUris());
  staticCommandInfo = builder.build();
}

Example #4

Source File: TestServiceCommandLine.java From incubator-myriad with Apache License 2.0

5 votes

@Test
public void testJHSCommandLineGeneration() throws Exception {
  Map<String, Long> portsMap = new TreeMap<>();
  portsMap.put(KEY_JHS_ADDRESS, 0L);
  portsMap.put(KEY_JHS_WEBAPP_ADDRESS, 3L);
  portsMap.put(KEY_JHS_ADMIN_ADDRESS, 0L);

  ServiceResourceProfile profile = new ServiceResourceProfile("jobhistory", 10.0, 15.0, portsMap);
  ServiceConfiguration serviceConfiguration = cfg.getServiceConfiguration("jobhistory").get();
  ServiceCommandLineGenerator serviceCommandLineGenerator = new ServiceCommandLineGenerator(cfg);
  List<Long> ports = new ArrayList<>();
  ports.add(2L);
  ports.add(1L);
  ports.add(3L);

  CommandInfo cInfo = serviceCommandLineGenerator.generateCommandLine(profile,
      serviceConfiguration,
      ports);
  String testVal =  String.format(CMD_FORMAT, toJHSCompare);
  assertTrue(String.format(msgFormat, cInfo.getValue(), testVal),
      cInfo.getValue().equals(testVal));

  List<Protos.Environment.Variable> environmentList = cInfo.getEnvironment().getVariablesList();
  String yarnOpts = "";
  for (Protos.Environment.Variable variable: environmentList) {
    if (variable.getName().equals(ServiceCommandLineGenerator.ENV_HADOOP_OPTS)){
      yarnOpts = variable.getValue();
    }
  }
  assertTrue("Environment contains " + ServiceCommandLineGenerator.ENV_HADOOP_OPTS, StringUtils.isNotEmpty(yarnOpts));
  System.out.println(yarnOpts);
  assertTrue(ServiceCommandLineGenerator.ENV_HADOOP_OPTS + " must contain -D" + KEY_JHS_WEBAPP_ADDRESS +
      "=0.0.0.0:3", yarnOpts.contains(KEY_JHS_WEBAPP_ADDRESS + "=0.0.0.0:3"));
}

Example #5

Source File: TestServiceCommandLine.java From incubator-myriad with Apache License 2.0

5 votes

@Test
public void testNMCommandLineGeneration() throws Exception {
  Long[] ports = new Long[]{1L, 2L, 3L, 4L};
  List<Long> nmPorts = Arrays.asList(ports);
  Map<String, Long> portsMap = new TreeMap<>();
  portsMap.put(KEY_NM_ADDRESS, 0L);
  portsMap.put(KEY_NM_WEBAPP_ADDRESS, 0L);
  portsMap.put(KEY_NM_LOCALIZER_ADDRESS, 0L);
  portsMap.put(KEY_NM_SHUFFLE_PORT, 0L);

  ServiceResourceProfile profile = new ExtendedResourceProfile(new NMProfile("nm", 10L, 15L), 3.0, 5.0, portsMap);

  ExecutorCommandLineGenerator clGenerator = new NMExecutorCommandLineGenerator(cfg);

  CommandInfo cInfo = clGenerator.generateCommandLine(profile, null, nmPorts);
  String testVal =  String.format(CMD_FORMAT, toCompare);
  assertTrue(String.format(msgFormat, cInfo.getValue(), testVal),
      cInfo.getValue().equals(testVal));

  List<Protos.Environment.Variable> environmentList = cInfo.getEnvironment().getVariablesList();
  String yarnOpts = "";
  for (Protos.Environment.Variable variable: environmentList) {
    if (variable.getName().equals(NMExecutorCommandLineGenerator.ENV_YARN_NODEMANAGER_OPTS)){
      yarnOpts = variable.getValue();
    }
  }
  System.out.println(yarnOpts);
  assertTrue("Environment contains " + NMExecutorCommandLineGenerator.ENV_YARN_NODEMANAGER_OPTS, StringUtils.isNotEmpty(yarnOpts));
  assertTrue(NMExecutorCommandLineGenerator.ENV_YARN_NODEMANAGER_OPTS + " must contain -D" + KEY_NM_SHUFFLE_PORT +
      "=1", yarnOpts.contains(KEY_NM_SHUFFLE_PORT + "=1"));
}

Example #6

Source File: ByteBufferSupportTest.java From incubator-myriad with Apache License 2.0

4 votes

private ExecutorInfo getExecutorInfo() {
  FrameworkID id = Protos.FrameworkID.newBuilder().setValue("framework1").build();
  ExecutorID eid = Protos.ExecutorID.newBuilder().setValue("executor1").build();
  CommandInfo cm = Protos.CommandInfo.newBuilder().setValue("command").build();
  return ExecutorInfo.newBuilder().setFrameworkId(id).setExecutorId(eid).setCommand(cm).build();
}

Example #7

Source File: TMSTaskFactoryImpl.java From incubator-myriad with Apache License 2.0

4 votes

@Override
public ExecutorInfo getExecutorInfoForSlave(ResourceOfferContainer resourceOfferContainer, FrameworkID frameworkId,
                                            CommandInfo commandInfo) {
  return null;
}

Example #8

Source File: REEFScheduler.java From reef with Apache License 2.0

4 votes

/**
 * Greedily acquire resources by launching a Mesos Task(w/ our custom MesosExecutor) on REEF Evaluator request.
 * Either called from onResourceRequest(for a new request) or resourceOffers(for an outstanding request).
 * TODO[JIRA REEF-102]: reflect priority and rack/node locality specified in resourceRequestEvent.
 */
private synchronized void doResourceRequest(final ResourceRequestEvent resourceRequestEvent) {
  int tasksToLaunchCounter = resourceRequestEvent.getResourceCount();

  for (final Offer offer : this.offers.values()) {
    final int cpuSlots = getCpu(offer) / resourceRequestEvent.getVirtualCores().get();
    final int memSlots = getMemory(offer) / resourceRequestEvent.getMemorySize().get();
    final int taskNum = Math.min(Math.min(cpuSlots, memSlots), tasksToLaunchCounter);

    if (taskNum > 0 && satisfySlaveConstraint(resourceRequestEvent, offer)) {
      final List<TaskInfo> tasksToLaunch = new ArrayList<>();
      tasksToLaunchCounter -= taskNum;

      // Launch as many MesosTasks on the same node(offer) as possible to exploit locality.
      for (int j = 0; j < taskNum; j++) {
        final String id = offer.getId().getValue() + "-" + String.valueOf(j);
        final String executorLaunchCommand = getExecutorLaunchCommand(id, resourceRequestEvent.getMemorySize().get());

        final ExecutorInfo executorInfo = ExecutorInfo.newBuilder()
            .setExecutorId(ExecutorID.newBuilder()
                .setValue(id)
                .build())
            .setCommand(CommandInfo.newBuilder()
                .setValue(executorLaunchCommand)
                .addUris(URI.newBuilder().setValue(reefTarUri).build())
                .build())
            .build();

        final TaskInfo taskInfo = TaskInfo.newBuilder()
            .setTaskId(TaskID.newBuilder()
                .setValue(id)
                .build())
            .setName(id)
            .setSlaveId(offer.getSlaveId())
            .addResources(Resource.newBuilder()
                    .setName("mem")
                    .setType(Type.SCALAR)
                    .setScalar(Value.Scalar.newBuilder()
                            .setValue(resourceRequestEvent.getMemorySize().get())
                            .build())
                    .build())
            .addResources(Resource.newBuilder()
                    .setName("cpus")
                    .setType(Type.SCALAR)
                    .setScalar(Value.Scalar.newBuilder()
                            .setValue(resourceRequestEvent.getVirtualCores().get())
                            .build())
                    .build())
            .setExecutor(executorInfo)
            .build();

        tasksToLaunch.add(taskInfo);
        this.executorIdToLaunchedRequests.put(id, resourceRequestEvent);
      }

      final Filters filters = Filters.newBuilder().setRefuseSeconds(0).build();
      mesosMaster.launchTasks(Collections.singleton(offer.getId()), tasksToLaunch, filters);
    } else {
      mesosMaster.declineOffer(offer.getId());
    }
  }

  // the offers are no longer valid(all launched or declined)
  this.offers.clear();

  // Save leftovers that couldn't be launched
  outstandingRequests.add(ResourceRequestEventImpl.newBuilder()
      .mergeFrom(resourceRequestEvent)
      .setResourceCount(tasksToLaunchCounter)
      .build());
}

Example #9

Source File: MesosNimbus.java From storm with Apache License 2.0

4 votes

private ExecutorInfo.Builder getExecutorInfoBuilder(TopologyDetails details, String executorDataStr,
                                                    String executorName,
                                                    List<Resource> executorResources,
                                                    String extraConfig) {
  String configUri;

  configUri = getFullConfigUri();

  ExecutorInfo.Builder executorInfoBuilder = ExecutorInfo.newBuilder();

  executorInfoBuilder
    .setName(executorName)
    .setExecutorId(ExecutorID.newBuilder().setValue(details.getId()))
    .setData(ByteString.copyFromUtf8(executorDataStr))
    .addAllResources(executorResources);

  ICommandLineShim commandLineShim = CommandLineShimFactory.makeCommandLineShim(_container.isPresent(), extraConfig);
  /**
   *  _container.isPresent() might be slightly misleading at first blush. It is only checking whether or not
   *  CONF_MESOS_CONTAINER_DOCKER_IMAGE is set to a value other than null.
   */
  if (_container.isPresent()) {
    executorInfoBuilder.setCommand(CommandInfo.newBuilder()
                                              .addUris(URI.newBuilder().setValue(configUri))
                                              .setValue(commandLineShim.getCommandLine(details.getId())))
                       .setContainer(ContainerInfo.newBuilder()
                                                  .setType(ContainerInfo.Type.DOCKER)
                                                  .setDocker(ContainerInfo.DockerInfo.newBuilder()
                                                                                     .setImage(_container.get())
                                                                                     .setNetwork(ContainerInfo.DockerInfo.Network.HOST)
                                                                                     .setForcePullImage(true)
                                                                                     .build()
                                                  ).build());
  } else {
    executorInfoBuilder.setCommand(CommandInfo.newBuilder()
                                              .addUris(URI.newBuilder().setValue((String) mesosStormConf.get(CONF_EXECUTOR_URI)))
                                              .addUris(URI.newBuilder().setValue(configUri))
                                              .setValue(commandLineShim.getCommandLine(details.getId())));
  }

  return executorInfoBuilder;
}

Example #10

Source File: TaskUtils.java From myriad with Apache License 2.0

4 votes

public static TaskInfo createYARNTask(Offer offer, NodeTask nodeTask) {
	NMProfile profile = nodeTask.getProfile();
	TaskID taskId = TaskID.newBuilder().setValue(nodeTask.getTaskId())
			.build();

	String taskIdValue = taskId.getValue();
	LOGGER.info("Launching task {} with profile: {}", taskIdValue, profile);
	String revisedConfig = getRevisedConfig(profile.getCpus(),
			profile.getMemory());
	String CONFIG_UPDATE_CMD = "sudo -H -u hduser bash -c \"echo '"
			+ revisedConfig
			+ "' > /usr/local/hadoop/etc/hadoop/yarn-site.xml\"; ";

	CommandInfo.Builder commandBuilder = CommandInfo.newBuilder();
	commandBuilder.setUser("root").setValue(
			CONFIG_UPDATE_CMD + NM_LAUNCH_CMD);

	TaskInfo task = TaskInfo
			.newBuilder()
			.setName("task " + taskIdValue)
			.setTaskId(taskId)
			.setSlaveId(offer.getSlaveId())
			.addResources(
					Resource.newBuilder()
							.setName("cpus")
							.setType(Value.Type.SCALAR)
							.setScalar(
									Value.Scalar.newBuilder()
											.setValue(profile.getCpus())
											.build()).build())
			.addResources(
					Resource.newBuilder()
							.setName("mem")
							.setType(Value.Type.SCALAR)
							.setScalar(
									Value.Scalar.newBuilder()
											.setValue(profile.getMemory())
											.build()).build())
			.setCommand(commandBuilder.build()).build();

	return task;
}

Example #11

Source File: ResourceMesosSchedulerFactory.java From oodt with Apache License 2.0

4 votes

public Scheduler construct() {
    try {
        String uri = System.getProperty("org.apache.oodt.cas.resource.mesos.executor.uri","./oodt-executor.in");
        //Framework info
        FrameworkInfo.Builder frameworkBuilder = FrameworkInfo.newBuilder()
                    .setName("OODT Resource Manager Mesos Framework").setUser("")
                    .setId(FrameworkID.newBuilder().setValue("OODT-Resource Framework").build());
        FrameworkInfo framework = frameworkBuilder.build();
        ExecutorInfo executor = ExecutorInfo.newBuilder().setExecutorId(ExecutorID.newBuilder().setValue("OODT-Resource").build())
                .setCommand(CommandInfo.newBuilder().setValue(new File(uri).getCanonicalPath()).build())
                .setName("OODT Resource Manager Executor").build();
        SchedulerDriver driver = null;

        //Resource manager properties
        String batchmgrClassStr = "org.apache.oodt.cas.resource.batchmgr.MesosBatchManagerFactory";
        String monitorClassStr = "org.apache.oodt.cas.resource.monitor.MesosMonitorFactory";
        String jobQueueClassStr = System.getProperty("resource.jobqueue.factory","org.apache.oodt.cas.resource.jobqueue.JobStackJobQueueFactory");
        String ip = System.getProperty("resource.mesos.master.ip","127.0.0.1:5050");

        batch = (MesosBatchManager)GenericResourceManagerObjectFactory.getBatchmgrServiceFromFactory(batchmgrClassStr);
        mon = GenericResourceManagerObjectFactory.getMonitorServiceFromFactory(monitorClassStr);
        queue = GenericResourceManagerObjectFactory.getJobQueueServiceFromFactory(jobQueueClassStr);
        batch.setMonitor(mon);
        batch.setDriver(driver);
        batch.setJobRepository(queue.getJobRepository());

        LOG.log(Level.INFO,"Connecting to Mesos Master at: "+ip);
        System.out.println("Connecting to Mesos Master at: "+ip);
        ResourceMesosScheduler scheduler = new ResourceMesosScheduler(batch, executor, queue, mon);

        final MesosSchedulerDriver mesos = new MesosSchedulerDriver(scheduler, framework, ip);
        //Anonymous thread to run
        new Thread(new Runnable() {
            public void run() {
                int status = mesos.run() == Status.DRIVER_STOPPED ? 0 : 1;
                mesos.stop();
            }
        }).start();
        return scheduler;
    } catch(IOException ioe) {
        LOG.log(Level.SEVERE,"Exception detected: "+ioe.getMessage());
        ioLOG.log(Level.SEVERE, e.getMessage());
        throw new RuntimeException(ioe);
    }
}

Example #12

Source File: HelloWorldMain.java From tutorials with MIT License

3 votes

public static void main(String[] args) {

        String path = System.getProperty("user.dir")
                + "/target/libraries2-1.0.0-SNAPSHOT.jar";

        CommandInfo.URI uri = CommandInfo.URI.newBuilder().setValue(path).setExtract(false).build();

        String helloWorldCommand = "java -cp libraries2-1.0.0-SNAPSHOT.jar com.baeldung.mesos.executors.HelloWorldExecutor";
        CommandInfo commandInfoHelloWorld = CommandInfo.newBuilder().setValue(helloWorldCommand).addUris(uri)
                .build();

        ExecutorInfo executorHelloWorld = ExecutorInfo.newBuilder()
                .setExecutorId(Protos.ExecutorID.newBuilder().setValue("HelloWorldExecutor"))
                .setCommand(commandInfoHelloWorld).setName("Hello World (Java)").setSource("java").build();

        FrameworkInfo.Builder frameworkBuilder = FrameworkInfo.newBuilder().setFailoverTimeout(120000)
                .setUser("")
                .setName("Hello World Framework (Java)");

        frameworkBuilder.setPrincipal("test-framework-java");

        MesosSchedulerDriver driver = new MesosSchedulerDriver(new HelloWorldScheduler(executorHelloWorld), frameworkBuilder.build(), args[0]);

        int status = driver.run() == Protos.Status.DRIVER_STOPPED ? 0 : 1;

        // Ensure that the driver process terminates.
        driver.stop();

        System.exit(status);
    }