org.apache.mesos.Protos.TaskState Java Examples

@Override
public void taskFinished(Task task, org.bds.task.TaskState taskState) {
	String tid = task.getId();
	if (debug) Gpr.debug("Task " + tid + " finished");

	CmdInfo cmdInfo = cmdInfoById.get(tid);
	if (cmdInfo == null) {
		if (debug) Gpr.debug("Task '" + tid + "' not found");
		return;
	}

	// Change Mesos task status to FINISHED
	changeTaskState(cmdInfo.executorDriver, cmdInfo.taskInfo, TaskState.TASK_FINISHED);

	// Clean up
	cmdInfoById.remove(tid);
}
 

@Subscribe
public void offerAccepted(OfferAccepted accepted) {
  // Move the task to starting after a delay.
  executor.schedule(
      () -> {
        master.changeState(
            ProtosConversion.convert(accepted.task.getTaskId()), TaskState.TASK_STARTING);

        executor.schedule(
            () -> master.changeState(
                ProtosConversion.convert(accepted.task.getTaskId()), TaskState.TASK_RUNNING),
            1,
            TimeUnit.SECONDS);
      },
      1,
      TimeUnit.SECONDS);
}
 

public void cleanup(TaskState state) {
  ExtendedTaskState extendedTaskState = MesosUtils.fromTaskState(
    org.apache.mesos.v1.Protos.TaskState.valueOf(state.toString())
  );

  boolean cleanupAppTaskDirectory =
    !extendedTaskState.isFailed() &&
    !taskDefinition
      .getExecutorData()
      .getPreserveTaskSandboxAfterFinish()
      .orElse(Boolean.FALSE);
  boolean cleanupLogs = false; // Task has just died, so we don't want to delete logs yet.

  boolean isDocker = (taskInfo.hasContainer() && taskInfo.getContainer().hasDocker());

  taskCleanup.cleanup(cleanupAppTaskDirectory, cleanupLogs, isDocker);
}
 

private void onFinish(SingularityExecutorTask task, Protos.TaskState taskState) {
  processKiller.cancelDestroyFuture(task.getTaskId());

  tasks.remove(task.getTaskId());
  processRunningTasks.remove(task.getTaskId());
  processBuildingTasks.remove(task.getTaskId());

  task.cleanup(taskState);

  ListeningExecutorService executorService = taskToShellCommandPool.remove(
    task.getTaskId()
  );

  if (executorService != null) {
    executorService.shutdownNow();
    try {
      executorService.awaitTermination(5, TimeUnit.MILLISECONDS);
    } catch (InterruptedException e) {
      LOG.warn("Awaiting shutdown of shell executor service", e);
    }
  }

  logging.stopTaskLogger(task.getTaskId(), task.getLogbackLog());

  checkIdleExecutorShutdown(task.getDriver());
}
 

public static boolean isTerminal(TaskState taskState) {
    switch (taskState) {
        case TASK_STAGING:
        case TASK_STARTING:
        case TASK_RUNNING:
        case TASK_KILLING:
            return false;
        case TASK_FINISHED:
        case TASK_FAILED:
        case TASK_KILLED:
        case TASK_ERROR:
        case TASK_LOST:
        case TASK_DROPPED:
        case TASK_UNREACHABLE:
        case TASK_GONE:
        case TASK_GONE_BY_OPERATOR:
        case TASK_UNKNOWN:
            return true;
    }
    throw new IllegalArgumentException("Unknown Mesos task state: " + taskState);
}
 

private void onRuntimeError() {
  // Shutdown REEF Evaluator
  if (this.evaluatorProcess != null) {
    this.evaluatorProcess.destroy();
  }
  mesosExecutorDriver.sendStatusUpdate(TaskStatus.newBuilder()
      .setTaskId(TaskID.newBuilder()
          .setValue(mesosExecutorId)
          .build())
      .setState(TaskState.TASK_FAILED)
      .setMessage("Evaluator Process exited with status " + String.valueOf(evaluatorProcessExitValue))
      .build());

  // Shutdown Mesos Executor
  this.executorService.shutdown();
  this.mesosExecutorDriver.stop();
}
 

@Override
public void killedWorker(int port) {
  LOG.info("killedWorker: executor {} removing port {} assignment and sending " +
      "TASK_FINISHED update to Mesos", _executorId, port);
  TaskID taskId = _taskAssignments.deregister(port);
  if (taskId == null) {
    LOG.error("killedWorker: Executor {} failed to find TaskID for port {}, so not " +
        "issuing TaskStatus update to Mesos for this dead task.", _executorId, port);
    return;
  }
  TaskStatus status = TaskStatus.newBuilder()
      .setState(TaskState.TASK_FINISHED)
      .setTaskId(taskId)
      .build();
  _driver.sendStatusUpdate(status);
}
 

@Override
public void killTask(ExecutorDriver driver, TaskID taskId) {
  String taskIdString = taskId.toString();
  LOGGER.debug("killTask received for taskId: " + taskIdString);
  TaskStatus status;

  if (!taskIdString.contains(MyriadExecutorAuxService.YARN_CONTAINER_TASK_ID_PREFIX)) {
    // Inform mesos of killing all tasks corresponding to yarn containers that are
    // currently running 
    synchronized (containerIds) {
      for (String containerId : containerIds) {
        Protos.TaskID containerTaskId = Protos.TaskID.newBuilder().setValue(
            MyriadExecutorAuxService.YARN_CONTAINER_TASK_ID_PREFIX + containerId).build();
        status = TaskStatus.newBuilder().setTaskId(containerTaskId).setState(TaskState.TASK_KILLED).build();
        driver.sendStatusUpdate(status);
      }
    }

    // Now kill the node manager task
    status = TaskStatus.newBuilder().setTaskId(taskId).setState(TaskState.TASK_KILLED).build();
    driver.sendStatusUpdate(status);
    LOGGER.info("NodeManager shutdown after receiving KILL_TASK for taskId {}", taskIdString);
    Runtime.getRuntime().exit(0);

  } else {
    status = TaskStatus.newBuilder().setTaskId(taskId).setState(TaskState.TASK_KILLED).build();
    driver.sendStatusUpdate(status);
    synchronized (containerIds) {
      //Likely the container isn't in here, but just in case remove it.
      if (containerIds.remove(taskIdString.substring(MyriadExecutorAuxService.YARN_CONTAINER_FULL_PREFIX.length(),
          taskIdString.length()))) {
        LOGGER.debug("Removed taskId {} from containerIds", taskIdString);
      }
    }
    LOGGER.debug("Killing " + taskId);
  }
}
 

@Override
public void stopContainer(ContainerTerminationContext stopContainerContext) {
  ContainerId containerId = stopContainerContext.getContainerId();
  synchronized (containerIds) {
    containerIds.remove(containerId.toString());
  }
  sendStatus(stopContainerContext.getContainerId(), TaskState.TASK_FINISHED);
}
 

@Override
public void initializeContainer(ContainerInitializationContext initContainerContext) {
  ContainerId containerId = initContainerContext.getContainerId();
  synchronized (containerIds) {
    containerIds.add(containerId.toString());
  }
  sendStatus(containerId, TaskState.TASK_RUNNING);
}
 

@Test
public void assertLaunchTaskWithDaemonTaskAndJavaSimpleJob() {
    TaskInfo taskInfo = buildJavaTransientTaskInfo();
    TaskExecutor.TaskThread taskThread = new TaskExecutor().new TaskThread(executorDriver, taskInfo);
    taskThread.run();
    Mockito.verify(executorDriver).sendStatusUpdate(Protos.TaskStatus.newBuilder().setTaskId(taskInfo.getTaskId()).setState(TaskState.TASK_RUNNING).build());
    Mockito.verify(executorDriver).sendStatusUpdate(Protos.TaskStatus.newBuilder().setTaskId(taskInfo.getTaskId()).setState(TaskState.TASK_FINISHED).build());
}
 

private void sendStatus(ContainerId containerId, TaskState taskState) {
  Protos.TaskID taskId = Protos.TaskID.newBuilder().setValue(YARN_CONTAINER_TASK_ID_PREFIX + containerId.toString()).build();

  TaskStatus status = TaskStatus.newBuilder().setTaskId(taskId).setState(taskState).build();
  driver.sendStatusUpdate(status);
  LOGGER.debug("Sent status " + taskState + " for taskId " + taskId);
}
 

private void loseTask(final TaskInfo taskInfo, final String reason)
{
    final StatusUpdateMessage statusUpdate = StatusUpdateMessage.newBuilder()
        .setUpdate(StatusUpdate.newBuilder()
            .setFrameworkId(context.getFrameworkId())
            .setSlaveId(taskInfo.getSlaveId())
            .setExecutorId(taskInfo.getExecutor().getExecutorId())
            .setStatus(TaskStatus.newBuilder().setTaskId(taskInfo.getTaskId()).setState(TaskState.TASK_LOST).setMessage(reason))
            .setTimestamp(TimeUtil.currentTime())
            .setUuid(UUIDUtil.uuidBytes(UUID.randomUUID())))
        .build();

    eventBus.post(new StatusUpdateMessageEnvelope(context.getDriverUPID(), context.getDriverUPID(), statusUpdate));
}
 

/**
 * We assume a long-running Mesos Task that manages a REEF Evaluator process, leveraging Mesos Executor's interface.
 */
@Override
public void launchTask(final ExecutorDriver driver, final TaskInfo task) {
  driver.sendStatusUpdate(TaskStatus.newBuilder()
      .setTaskId(TaskID.newBuilder().setValue(this.mesosExecutorId).build())
      .setState(TaskState.TASK_STARTING)
      .setSlaveId(task.getSlaveId())
      .setMessage(this.mesosRemoteManager.getMyIdentifier())
      .build());
}
 

private void onStop() {
  // Shutdown REEF Evaluator
  if (this.evaluatorProcess != null) {
    this.evaluatorProcess.destroy();
    mesosExecutorDriver.sendStatusUpdate(TaskStatus.newBuilder()
        .setTaskId(TaskID.newBuilder()
            .setValue(mesosExecutorId)
            .build())
        .setState(TaskState.TASK_FINISHED)
        .setMessage("Evaluator Process exited with status " + String.valueOf(evaluatorProcessExitValue))
        .build());
  } else {
    mesosExecutorDriver.sendStatusUpdate(TaskStatus.newBuilder()
        .setTaskId(TaskID.newBuilder()
            .setValue(mesosExecutorId)
            .build())
        .setState(TaskState.TASK_FINISHED)
        .setData(ByteString.copyFromUtf8("eval_not_run"))
        // TODO[JIRA REEF-102]: a hack to pass closeEvaluator test, replace this with a better interface
        .setMessage("Evaluator Process exited with status " + String.valueOf(evaluatorProcessExitValue))
        .build());
  }

  // Shutdown Mesos Executor
  this.executorService.shutdown();
  this.mesosExecutorDriver.stop();
}
 

public void doRegistration(final SchedulerDriver driver, Protos.FrameworkID id) {
  _driver = driver;
  // Now that we've set the driver, we can create our scheduler
  _stormScheduler = new StormSchedulerImpl(_driver);

  _state.put(FRAMEWORK_ID, id.getValue());
  _offers = new HashMap<Protos.OfferID, Protos.Offer>();

  if (_enabledLogviewerSidecar) {

    _timer.scheduleAtFixedRate(new TimerTask() {
      @Override
      public void run() {
        // performing "explicit" reconciliation; master will respond with the latest state for all logviewer tasks
        // in the framework scheduler's statusUpdate() method
        List<TaskStatus> taskStatuses = new ArrayList<TaskStatus>();
        List<String> logviewerPaths = _zkClient.getChildren(_logviewerZkDir);
        if (logviewerPaths == null) {
          _driver.reconcileTasks(taskStatuses);
          return;
        }
        for (String path : logviewerPaths) {
          TaskID logviewerTaskId = TaskID.newBuilder()
                                         .setValue(new String(_zkClient.getNodeData(String.format("%s/%s", _logviewerZkDir, path))))
                                         .build();
          TaskStatus logviewerTaskStatus = TaskStatus.newBuilder()
                                                     .setTaskId(logviewerTaskId)
                                                     .setState(TaskState.TASK_RUNNING)
                                                     .build();
          taskStatuses.add(logviewerTaskStatus);
        }
        _driver.reconcileTasks(taskStatuses);
        LOG.info("Performing task reconciliation between scheduler and master on following tasks: {}", taskStatusListToTaskIDsString(taskStatuses));
      }
    }, 0, TASK_RECONCILIATION_INTERVAL); // reconciliation performed every 5 minutes
  }
}
 

@Override
public void statusUpdate(SchedulerDriver sd, Protos.TaskStatus status) {
	TaskID taskId = status.getTaskId();
	LOGGER.info("Status Update for task: {} | state: {}", taskId,
			status.getState());
	TaskState state = status.getState();

	String taskIdValue = taskId.getValue();
	switch (state) {
	case TASK_STAGING:
		schedulerState.makeTaskStaging(taskIdValue);
		break;
	case TASK_STARTING:
		schedulerState.makeTaskStaging(taskIdValue);
		break;
	case TASK_RUNNING:
		schedulerState.makeTaskActive(taskIdValue);
		NodeTask task = schedulerState.getTask(taskIdValue);
		schedulerState.releaseLock(task.getClusterId());
		break;
	case TASK_FINISHED:
		schedulerState.removeTask(taskIdValue);
		break;
	case TASK_FAILED:
		// Add to pending tasks
		schedulerState.makeTaskPending(taskIdValue);
		break;
	case TASK_KILLED:
		schedulerState.removeTask(taskIdValue);
		break;
	case TASK_LOST:
		schedulerState.makeTaskPending(taskIdValue);
		break;
	default:
		LOGGER.error("Invalid state: {}", state);
		break;
	}
}
 

public void finishTask(
  final SingularityExecutorTask task,
  Protos.TaskState taskState,
  String message,
  Optional<String> errorMsg,
  Object... errorObjects
) {
  try {
    if (errorMsg.isPresent()) {
      task.getLog().error(errorMsg.get(), errorObjects);
    }
  } finally {
    try {
      sendStatusUpdate(task, taskState, message);

      onFinish(task, taskState);
    } catch (Throwable t) {
      logAndExit(
        3,
        "Failed while finishing task {} (state {})",
        task.getTaskId(),
        taskState,
        t
      );
    }
  }
}
 

private void sendStatusUpdate(
  SingularityExecutorTask task,
  Protos.TaskState taskState,
  String message
) {
  executorUtils.sendStatusUpdate(
    task.getDriver(),
    TaskID.newBuilder().setValue(task.getTaskId()).build(),
    taskState,
    message,
    task.getLog()
  );
}
 

public void changeState(TaskID task, TaskState state) {
  assertNotStopped();

  checkState(activeTasks.containsKey(task), "Task " + task + " does not exist.");
  Futures.getUnchecked(schedulerFuture).statusUpdate(this, TaskStatus.newBuilder()
      .setTaskId(task)
      .setState(state)
      .build());
}
 

@Override
public Status launchTasks(Collection<OfferID> offerIds, Collection<TaskInfo> tasks) {
  assertNotStopped();

  OfferID id = Iterables.getOnlyElement(offerIds);
  Offer offer = sentOffers.remove(id);
  checkState(offer != null, "Offer " + id + " is invalid.");

  final TaskInfo task = Iterables.getOnlyElement(tasks);
  synchronized (activeTasks) {
    checkState(
        !activeTasks.containsKey(task.getTaskId()),
        "Task " + task.getTaskId() + " already exists.");
    activeTasks.put(task.getTaskId(), new Task(offer, task));
  }

  executor.schedule(
      () -> Futures.getUnchecked(schedulerFuture).statusUpdate(
          this,
          TaskStatus.newBuilder()
              .setTaskId(task.getTaskId())
              .setState(TaskState.TASK_RUNNING)
              .build()),
      1,
      TimeUnit.SECONDS);

  return Status.DRIVER_RUNNING;
}
 

@Override
public Status killTask(TaskID taskId) {
  assertNotStopped();

  Task task = activeTasks.remove(taskId);
  checkState(task != null, "Task " + taskId + " not found.");
  idleOffers.put(task.getOffer().getId(), task.getOffer());

  Futures.getUnchecked(schedulerFuture).statusUpdate(this, TaskStatus.newBuilder()
      .setTaskId(taskId)
      .setState(TaskState.TASK_FINISHED)
      .build());

  return Status.DRIVER_RUNNING;
}
 

@Override
public void taskRunning(Task task) {
	String tid = task.getId();
	if (debug) Gpr.debug("Task " + tid + " running");

	CmdInfo cmdInfo = cmdInfoById.get(tid);
	if (cmdInfo == null) {
		if (debug) Gpr.debug("Task '" + tid + "' not found");
		return;
	}

	// Change Mesos task status to FINISHED
	changeTaskState(cmdInfo.executorDriver, cmdInfo.taskInfo, TaskState.TASK_RUNNING);
}
 

public TaskState transitionToUnchecked(TaskState nextState, Protos.TaskStatus.Reason reason, String reasonMessage) {
    TaskState oldState = currentTaskStatus.getState();

    logger.info("Changing task state: taskId={}, from={}, to={}", taskId, oldState, nextState);

    Protos.TaskStatus.Builder statusBuilder = newTaskStatusBuilder()
            .setState(nextState)
            .setReason(reason)
            .setMessage(reasonMessage);

    if (nextState == TaskState.TASK_STARTING && containerIp != null) {
        TitusExecutorDetails details = new TitusExecutorDetails(
                Collections.singletonMap("nfvpc", containerIp),
                new TitusExecutorDetails.NetworkConfiguration(
                        true,
                        containerIp,
                        null,
                        containerIp,
                        "simulatedENI-" + eniID,
                        "resource-eni-" + eniID
                )
        );
        try {
            statusBuilder.setData(ByteString.copyFrom(ObjectMappers.compactMapper().writeValueAsString(details), Charset.defaultCharset()));
        } catch (Exception e) {
            // IGNORE
        }
    }

    currentTaskStatus = statusBuilder.build();

    long delay = delayFunction.apply(nextState);
    if (delay <= 0) {
        emitTaskStateUpdate();
    } else {
        Observable.timer(delay, TimeUnit.MILLISECONDS).subscribe(tick -> emitTaskStateUpdate());
    }

    return oldState;
}
 

@Test
public void assertLaunchTaskWithTransientTaskAndSpringSimpleJob() {
    TaskInfo taskInfo = buildSpringDaemonTaskInfo();
    TaskExecutor.TaskThread taskThread = new TaskExecutor().new TaskThread(executorDriver, taskInfo);
    taskThread.run();
    Mockito.verify(executorDriver).sendStatusUpdate(Protos.TaskStatus.newBuilder().setTaskId(taskInfo.getTaskId()).setState(TaskState.TASK_RUNNING).build());
}
 

@Test
public void assertLaunchTaskWithDaemonTaskAndJavaScriptJob() {
    TaskInfo taskInfo = buildSpringScriptTransientTaskInfo();
    TaskExecutor.TaskThread taskThread = new TaskExecutor().new TaskThread(executorDriver, taskInfo);
    taskThread.run();
    Mockito.verify(executorDriver).sendStatusUpdate(Protos.TaskStatus.newBuilder().setTaskId(taskInfo.getTaskId()).setState(TaskState.TASK_RUNNING).build());
    Mockito.verify(executorDriver).sendStatusUpdate(Protos.TaskStatus.newBuilder().setTaskId(taskInfo.getTaskId()).setState(TaskState.TASK_FINISHED).build());
}
 

@Test
public void assertJobRun() throws Exception {
    when(jobFacade.getShardingContexts()).thenReturn(shardingContexts);
    when(jobFacade.loadJobRootConfiguration(true)).thenReturn(new TestScriptJobConfiguration("test.sh"));
    daemonJob.execute(jobExecutionContext);
    verify(shardingContexts).setAllowSendJobEvent(true);
    verify(executorDriver).sendStatusUpdate(TaskStatus.newBuilder().setTaskId(taskId).setState(TaskState.TASK_RUNNING).setMessage("BEGIN").build());
    verify(executorDriver).sendStatusUpdate(TaskStatus.newBuilder().setTaskId(taskId).setState(TaskState.TASK_RUNNING).setMessage("COMPLETE").build());
    verify(shardingContexts).setCurrentJobEventSamplingCount(0);
}
 

@Test
public void assertJobRunWithEventSampling() throws Exception {
    when(shardingContexts.getJobEventSamplingCount()).thenReturn(2);
    when(jobFacade.getShardingContexts()).thenReturn(shardingContexts);
    when(jobFacade.loadJobRootConfiguration(true)).thenReturn(new TestScriptJobConfiguration("test.sh"));
    daemonJob.execute(jobExecutionContext);
    verify(shardingContexts).setCurrentJobEventSamplingCount(1);
    verify(shardingContexts).setAllowSendJobEvent(false);
    when(shardingContexts.getCurrentJobEventSamplingCount()).thenReturn(1);
    daemonJob.execute(jobExecutionContext);
    verify(shardingContexts).setAllowSendJobEvent(true);
    verify(executorDriver).sendStatusUpdate(TaskStatus.newBuilder().setTaskId(taskId).setState(TaskState.TASK_RUNNING).setMessage("BEGIN").build());
    verify(executorDriver).sendStatusUpdate(TaskStatus.newBuilder().setTaskId(taskId).setState(TaskState.TASK_RUNNING).setMessage("COMPLETE").build());
    verify(shardingContexts).setCurrentJobEventSamplingCount(0);
}
 

public TaskExecutorHolder(ContainerPlayersManager containerPlayersManager,
                          String jobId,
                          String taskId,
                          SimulatedTitusAgent agent,
                          AwsInstanceType instanceType,
                          double taskCPUs,
                          double taskGPUs,
                          double taskMem,
                          double taskDisk,
                          Set<Long> allocatedPorts,
                          String containerIp,
                          String eniID,
                          double taskNetworkMbs,
                          List<EfsMount> efsMounts,
                          Map<String, String> env,
                          Observer<Protos.TaskStatus> stateUpdatesObserver) {
    this.jobId = jobId;
    this.taskId = taskId;
    this.agent = agent;
    this.instanceType = instanceType;
    this.taskCPUs = taskCPUs;
    this.taskGPUs = taskGPUs;
    this.taskMem = taskMem;
    this.taskDisk = taskDisk;
    this.allocatedPorts = allocatedPorts;
    this.containerIp = containerIp;
    this.eniID = eniID;
    this.taskNetworkMbs = taskNetworkMbs;
    this.efsMounts = efsMounts;
    this.env = env;
    this.stateUpdatesObserver = stateUpdatesObserver;
    this.currentTaskStatus = newTaskStatusBuilder().setState(TaskState.TASK_STAGING).setMessage("Task staging").build();
    this.delayFunction = taskState -> 0L; // No transition delay by default

    containerPlayersManager.play(this);

    stateUpdatesObserver.onNext(currentTaskStatus);
}
 

public TaskState transitionTo(TaskState nextState, Protos.TaskStatus.Reason reason, String reasonMessage) {
    TaskState currentTaskState = currentTaskStatus.getState();
    if (!VALID_STATE_TRANSITIONS.get(currentTaskState).contains(nextState) && !isTerminal(nextState)) {
        throw new IllegalArgumentException("State transition " + currentTaskState + " -> " + nextState + " not allowed");
    }
    return transitionToUnchecked(nextState, reason, reasonMessage);
}