Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt#getMasterContainer()

public AppAttemptInfo(RMAppAttempt attempt, String user) {
  this.startTime = 0;
  this.containerId = "";
  this.nodeHttpAddress = "";
  this.nodeId = "";
  this.logsLink = "";
  if (attempt != null) {
    this.id = attempt.getAppAttemptId().getAttemptId();
    this.startTime = attempt.getStartTime();
    Container masterContainer = attempt.getMasterContainer();
    if (masterContainer != null) {
      this.containerId = masterContainer.getId().toString();
      this.nodeHttpAddress = masterContainer.getNodeHttpAddress();
      this.nodeId = masterContainer.getNodeId().toString();
      this.logsLink =
          WebAppUtils.getRunningLogURL("//" + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()), user);
    }
  }
}
 

public AppAttemptInfo(RMAppAttempt attempt, String user) {
  this.startTime = 0;
  this.containerId = "";
  this.nodeHttpAddress = "";
  this.nodeId = "";
  this.logsLink = "";
  if (attempt != null) {
    this.id = attempt.getAppAttemptId().getAttemptId();
    this.startTime = attempt.getStartTime();
    Container masterContainer = attempt.getMasterContainer();
    if (masterContainer != null) {
      this.containerId = masterContainer.getId().toString();
      this.nodeHttpAddress = masterContainer.getNodeHttpAddress();
      this.nodeId = masterContainer.getNodeId().toString();
      this.logsLink =
          WebAppUtils.getRunningLogURL("//" + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()), user);
    }
  }
}
 

/**
 * Helper method to handle received ContainerStatus. If this corresponds to
 * the completion of a master-container of a managed AM,
 * we call the handler for RMAppAttemptContainerFinishedEvent.
 */
@SuppressWarnings("unchecked")
@VisibleForTesting
void handleNMContainerStatus(NMContainerStatus containerStatus, NodeId nodeId) {
  ApplicationAttemptId appAttemptId =
      containerStatus.getContainerId().getApplicationAttemptId();
  RMApp rmApp =
      rmContext.getRMApps().get(appAttemptId.getApplicationId());
  if (rmApp == null) {
    LOG.error("Received finished container : "
        + containerStatus.getContainerId()
        + " for unknown application " + appAttemptId.getApplicationId()
        + " Skipping.");
    return;
  }

  if (rmApp.getApplicationSubmissionContext().getUnmanagedAM()) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Ignoring container completion status for unmanaged AM "
          + rmApp.getApplicationId());
    }
    return;
  }

  RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptId);
  Container masterContainer = rmAppAttempt.getMasterContainer();
  if (masterContainer.getId().equals(containerStatus.getContainerId())
      && containerStatus.getContainerState() == ContainerState.COMPLETE) {
    ContainerStatus status =
        ContainerStatus.newInstance(containerStatus.getContainerId(),
          containerStatus.getContainerState(), containerStatus.getDiagnostics(),
          containerStatus.getContainerExitStatus());
    // sending master container finished event.
    RMAppAttemptContainerFinishedEvent evt =
        new RMAppAttemptContainerFinishedEvent(appAttemptId, status,
            nodeId);
    rmContext.getDispatcher().getEventHandler().handle(evt);
  }
}
 

public AMLauncher(RMContext rmContext, RMAppAttempt application,
    AMLauncherEventType eventType, Configuration conf) {
  this.application = application;
  this.conf = conf;
  this.eventType = eventType;
  this.rmContext = rmContext;
  this.handler = rmContext.getDispatcher().getEventHandler();
  this.masterContainer = application.getMasterContainer();
}
 

/**
 * Helper method to handle received ContainerStatus. If this corresponds to
 * the completion of a master-container of a managed AM,
 * we call the handler for RMAppAttemptContainerFinishedEvent.
 */
@SuppressWarnings("unchecked")
@VisibleForTesting
void handleNMContainerStatus(NMContainerStatus containerStatus, NodeId nodeId) {
  ApplicationAttemptId appAttemptId =
      containerStatus.getContainerId().getApplicationAttemptId();
  RMApp rmApp =
      rmContext.getRMApps().get(appAttemptId.getApplicationId());
  if (rmApp == null) {
    LOG.error("Received finished container : "
        + containerStatus.getContainerId()
        + " for unknown application " + appAttemptId.getApplicationId()
        + " Skipping.");
    return;
  }

  if (rmApp.getApplicationSubmissionContext().getUnmanagedAM()) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Ignoring container completion status for unmanaged AM "
          + rmApp.getApplicationId());
    }
    return;
  }

  RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptId);
  Container masterContainer = rmAppAttempt.getMasterContainer();
  if (masterContainer.getId().equals(containerStatus.getContainerId())
      && containerStatus.getContainerState() == ContainerState.COMPLETE) {
    ContainerStatus status =
        ContainerStatus.newInstance(containerStatus.getContainerId(),
          containerStatus.getContainerState(), containerStatus.getDiagnostics(),
          containerStatus.getContainerExitStatus());
    // sending master container finished event.
    RMAppAttemptContainerFinishedEvent evt =
        new RMAppAttemptContainerFinishedEvent(appAttemptId, status,
            nodeId);
    rmContext.getDispatcher().getEventHandler().handle(evt);
  }
}
 

public AMLauncher(RMContext rmContext, RMAppAttempt application,
    AMLauncherEventType eventType, Configuration conf) {
  this.application = application;
  this.conf = conf;
  this.eventType = eventType;
  this.rmContext = rmContext;
  this.handler = rmContext.getDispatcher().getEventHandler();
  this.masterContainer = application.getMasterContainer();
}
 

public void sendAppEvent(ApplicationId applicationId, RMApp rmApp) {
    if (cacheFinishedApp.getIfPresent(applicationId.toString()) == null) {
        Header.Builder headerBuilder = Header.newBuilder()
            .withId(applicationId.toString())
            .withApplicationID(applicationId.toString())
            .withUser(rmApp.getUser())
            .withApplicationName(rmApp.getName())
            .withFramework(rmApp.getApplicationType().toUpperCase());

        ApplicationEvent.Builder eventBuilder = ApplicationEvent.newBuilder()
            .setState(rmApp.getState().name())
            .setQueue(rmApp.getQueue());

        rmApp.getApplicationTags().stream()
            .filter(tag -> YARN_TAGS_TO_EXTRACT.stream().noneMatch(tag::startsWith) && !tag.contains(":"))
            .forEach(eventBuilder::addYarnTags);

        rmApp.getApplicationTags().stream()
            .filter(tag -> tag.contains(":") && YARN_TAGS_TO_EXTRACT.stream().anyMatch(tag::startsWith))
            .map(tag -> {
                int idx = tag.indexOf(':');
                String key = tag.substring(0, idx);
                String value = tag.substring(idx + 1);
                return new String[] {key, value};
            })
            .forEach(splitTag -> BUILDERS.get(splitTag[0]).accept(splitTag[1], eventBuilder));

        eventBuilder.setFinalStatus(rmApp.getFinalApplicationStatus().name());

        eventBuilder.setStartTime(rmApp.getStartTime());
        eventBuilder.setFinishTime(rmApp.getFinishTime());

        RMAppMetrics rmAppMetrics = rmApp.getRMAppMetrics();
        if (rmAppMetrics != null) {
            eventBuilder.setMemorySeconds(rmAppMetrics.getMemorySeconds());
            eventBuilder.setVcoreSeconds(rmAppMetrics.getVcoreSeconds());
        }

        RMAppAttempt rmAppAttempt = rmApp.getCurrentAppAttempt();
        if (rmAppAttempt != null) {
            headerBuilder.withAttemptID(rmAppAttempt.getAppAttemptId().toString());

            Container container = rmAppAttempt.getMasterContainer();
            if (container != null) {
                eventBuilder.setAmContainerId(container.getId().toString());
            }
        }

        if (rmApp.getTrackingUrl() != null) {
            eventBuilder.setTrackingUrl(normalizeTrackingUrl(rmApp.getTrackingUrl()));
        }

        if (rmApp.getOriginalTrackingUrl() != null && !"N/A".equals(rmApp.getOriginalTrackingUrl())) {
            eventBuilder.setOriginalTrackingUrl(normalizeTrackingUrl(rmApp.getOriginalTrackingUrl()));
        }

        eventHandler.accept(System.currentTimeMillis(), headerBuilder.build(), eventBuilder.build());

        if (rmApp.getState() == RMAppState.FINISHED || rmApp.getState() == RMAppState.KILLED || rmApp.getState() == RMAppState.FAILED) {
            cacheFinishedApp.put(applicationId.toString(), rmApp.getState().name());
        }
    }
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          allocatedGCores = usedResources.getGpuCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    preemptedResourceGCores =
        appMetrics.getResourcePreempted().getGpuCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
    gcoreSeconds = appMetrics.getGcoreSeconds();
  }
}
 

public synchronized void recoverContainersOnNode(
    List<NMContainerStatus> containerReports, RMNode nm) {
  if (!rmContext.isWorkPreservingRecoveryEnabled()
      || containerReports == null
      || (containerReports != null && containerReports.isEmpty())) {
    return;
  }

  for (NMContainerStatus container : containerReports) {
    ApplicationId appId =
        container.getContainerId().getApplicationAttemptId().getApplicationId();
    RMApp rmApp = rmContext.getRMApps().get(appId);
    if (rmApp == null) {
      LOG.error("Skip recovering container " + container
          + " for unknown application.");
      killOrphanContainerOnNode(nm, container);
      continue;
    }

    // Unmanaged AM recovery is addressed in YARN-1815
    if (rmApp.getApplicationSubmissionContext().getUnmanagedAM()) {
      LOG.info("Skip recovering container " + container + " for unmanaged AM."
          + rmApp.getApplicationId());
      killOrphanContainerOnNode(nm, container);
      continue;
    }

    SchedulerApplication<T> schedulerApp = applications.get(appId);
    if (schedulerApp == null) {
      LOG.info("Skip recovering container  " + container
          + " for unknown SchedulerApplication. Application current state is "
          + rmApp.getState());
      killOrphanContainerOnNode(nm, container);
      continue;
    }

    LOG.info("Recovering container " + container);
    SchedulerApplicationAttempt schedulerAttempt =
        schedulerApp.getCurrentAppAttempt();

    if (!rmApp.getApplicationSubmissionContext()
      .getKeepContainersAcrossApplicationAttempts()) {
      // Do not recover containers for stopped attempt or previous attempt.
      if (schedulerAttempt.isStopped()
          || !schedulerAttempt.getApplicationAttemptId().equals(
            container.getContainerId().getApplicationAttemptId())) {
        LOG.info("Skip recovering container " + container
            + " for already stopped attempt.");
        killOrphanContainerOnNode(nm, container);
        continue;
      }
    }

    // create container
    RMContainer rmContainer = recoverAndCreateContainer(container, nm);

    // recover RMContainer
    rmContainer.handle(new RMContainerRecoverEvent(container.getContainerId(),
      container));

    // recover scheduler node
    SchedulerNode schedulerNode = nodes.get(nm.getNodeID());
    schedulerNode.recoverContainer(rmContainer);

    // recover queue: update headroom etc.
    Queue queue = schedulerAttempt.getQueue();
    queue.recoverContainer(clusterResource, schedulerAttempt, rmContainer);

    // recover scheduler attempt
    schedulerAttempt.recoverContainer(schedulerNode, rmContainer);
          
    // set master container for the current running AMContainer for this
    // attempt.
    RMAppAttempt appAttempt = rmApp.getCurrentAppAttempt();
    if (appAttempt != null) {
      Container masterContainer = appAttempt.getMasterContainer();

      // Mark current running AMContainer's RMContainer based on the master
      // container ID stored in AppAttempt.
      if (masterContainer != null
          && masterContainer.getId().equals(rmContainer.getContainerId())) {
        ((RMContainerImpl)rmContainer).setAMContainer(true);
      }
    }

    synchronized (schedulerAttempt) {
      Set<ContainerId> releases = schedulerAttempt.getPendingRelease();
      if (releases.contains(container.getContainerId())) {
        // release the container
        rmContainer.handle(new RMContainerFinishedEvent(container
          .getContainerId(), SchedulerUtils.createAbnormalContainerStatus(
          container.getContainerId(), SchedulerUtils.RELEASED_CONTAINER),
          RMContainerEventType.RELEASED));
        releases.remove(container.getContainerId());
        LOG.info(container.getContainerId() + " is released by application.");
      }
    }
  }
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
  }
}
 

public synchronized void recoverContainersOnNode(
    List<NMContainerStatus> containerReports, RMNode nm) {
  if (!rmContext.isWorkPreservingRecoveryEnabled()
      || containerReports == null
      || (containerReports != null && containerReports.isEmpty())) {
    return;
  }

  for (NMContainerStatus container : containerReports) {
    ApplicationId appId =
        container.getContainerId().getApplicationAttemptId().getApplicationId();
    RMApp rmApp = rmContext.getRMApps().get(appId);
    if (rmApp == null) {
      LOG.error("Skip recovering container " + container
          + " for unknown application.");
      killOrphanContainerOnNode(nm, container);
      continue;
    }

    // Unmanaged AM recovery is addressed in YARN-1815
    if (rmApp.getApplicationSubmissionContext().getUnmanagedAM()) {
      LOG.info("Skip recovering container " + container + " for unmanaged AM."
          + rmApp.getApplicationId());
      killOrphanContainerOnNode(nm, container);
      continue;
    }

    SchedulerApplication<T> schedulerApp = applications.get(appId);
    if (schedulerApp == null) {
      LOG.info("Skip recovering container  " + container
          + " for unknown SchedulerApplication. Application current state is "
          + rmApp.getState());
      killOrphanContainerOnNode(nm, container);
      continue;
    }

    LOG.info("Recovering container " + container);
    SchedulerApplicationAttempt schedulerAttempt =
        schedulerApp.getCurrentAppAttempt();

    if (!rmApp.getApplicationSubmissionContext()
      .getKeepContainersAcrossApplicationAttempts()) {
      // Do not recover containers for stopped attempt or previous attempt.
      if (schedulerAttempt.isStopped()
          || !schedulerAttempt.getApplicationAttemptId().equals(
            container.getContainerId().getApplicationAttemptId())) {
        LOG.info("Skip recovering container " + container
            + " for already stopped attempt.");
        killOrphanContainerOnNode(nm, container);
        continue;
      }
    }

    // create container
    RMContainer rmContainer = recoverAndCreateContainer(container, nm);

    // recover RMContainer
    rmContainer.handle(new RMContainerRecoverEvent(container.getContainerId(),
      container));

    // recover scheduler node
    nodes.get(nm.getNodeID()).recoverContainer(rmContainer);

    // recover queue: update headroom etc.
    Queue queue = schedulerAttempt.getQueue();
    queue.recoverContainer(clusterResource, schedulerAttempt, rmContainer);

    // recover scheduler attempt
    schedulerAttempt.recoverContainer(rmContainer);
          
    // set master container for the current running AMContainer for this
    // attempt.
    RMAppAttempt appAttempt = rmApp.getCurrentAppAttempt();
    if (appAttempt != null) {
      Container masterContainer = appAttempt.getMasterContainer();

      // Mark current running AMContainer's RMContainer based on the master
      // container ID stored in AppAttempt.
      if (masterContainer != null
          && masterContainer.getId().equals(rmContainer.getContainerId())) {
        ((RMContainerImpl)rmContainer).setAMContainer(true);
      }
    }

    synchronized (schedulerAttempt) {
      Set<ContainerId> releases = schedulerAttempt.getPendingRelease();
      if (releases.contains(container.getContainerId())) {
        // release the container
        rmContainer.handle(new RMContainerFinishedEvent(container
          .getContainerId(), SchedulerUtils.createAbnormalContainerStatus(
          container.getContainerId(), SchedulerUtils.RELEASED_CONTAINER),
          RMContainerEventType.RELEASED));
        releases.remove(container.getContainerId());
        LOG.info(container.getContainerId() + " is released by application.");
      }
    }
  }
}