org.apache.hadoop.yarn.util.RackResolver Java Examples

HeldContainer(Container container,
    long nextScheduleTime,
    long containerExpiryTime,
    CookieContainerRequest firstTaskInfo,
    ContainerSignatureMatcher signatureMatcher) {
  this.container = container;
  this.nextScheduleTime = nextScheduleTime;
  if (firstTaskInfo != null) {
    this.lastTaskInfo = firstTaskInfo;
    this.lastAssignedContainerSignature = firstTaskInfo.getCookie().getContainerSignature();
  }
  this.localityMatchLevel = LocalityMatchLevel.NODE;
  this.containerExpiryTime = containerExpiryTime;
  this.rack = RackResolver.resolve(container.getNodeId().getHost())
      .getNetworkLocation();
  this.signatureMatcher = signatureMatcher;
}
 

private Set<String> resolveRacks(List<String> nodes) {
  Set<String> racks = new HashSet<String>();    
  if (nodes != null) {
    for (String node : nodes) {
      // Ensure node requests are accompanied by requests for
      // corresponding rack
      String rack = RackResolver.resolve(node).getNetworkLocation();
      if (rack == null) {
        LOG.warn("Failed to resolve rack for node " + node + ".");
      } else {
        racks.add(rack);
      }
    }
  }
  
  return racks;
}
 

private void computeRackAndLocality() {
  NodeId containerNodeId = container.getNodeId();
  nodeRackName = RackResolver.resolve(
      containerNodeId.getHost()).getNetworkLocation();

  locality = Locality.OFF_SWITCH;
  if (dataLocalHosts.size() > 0) {
    String cHost = resolveHost(containerNodeId.getHost());
    if (dataLocalHosts.contains(cHost)) {
      locality = Locality.NODE_LOCAL;
    }
  }
  if (locality == Locality.OFF_SWITCH) {
    if (dataLocalRacks.contains(nodeRackName)) {
      locality = Locality.RACK_LOCAL;
    }
  }
}
 

private void computeRackAndLocality() {
  NodeId containerNodeId = container.getNodeId();
  nodeRackName = RackResolver.resolve(
      containerNodeId.getHost()).getNetworkLocation();

  locality = Locality.OFF_SWITCH;
  if (dataLocalHosts.size() > 0) {
    String cHost = resolveHost(containerNodeId.getHost());
    if (dataLocalHosts.contains(cHost)) {
      locality = Locality.NODE_LOCAL;
    }
  }
  if (locality == Locality.OFF_SWITCH) {
    if (dataLocalRacks.contains(nodeRackName)) {
      locality = Locality.RACK_LOCAL;
    }
  }
}
 

@SuppressWarnings("rawtypes")
public TaskAttemptImpl(TezTaskID taskId, int attemptNumber, EventHandler eventHandler,
    TaskAttemptListener taskAttemptListener, Configuration conf, Clock clock,
    TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext,
    boolean isRescheduled,
    Resource resource, ContainerContext containerContext, boolean leafVertex) {
  ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
  this.readLock = rwLock.readLock();
  this.writeLock = rwLock.writeLock();
  this.attemptId = TezBuilderUtils.newTaskAttemptId(taskId, attemptNumber);
  this.eventHandler = eventHandler;
  //Reported status
  this.conf = conf;
  this.clock = clock;
  this.taskHeartbeatHandler = taskHeartbeatHandler;
  this.appContext = appContext;
  this.reportedStatus = new TaskAttemptStatus();
  initTaskAttemptStatus(reportedStatus);
  RackResolver.init(conf);
  this.stateMachine = stateMachineFactory.make(this);
  this.isRescheduled = isRescheduled;
  this.taskResource = resource;
  this.containerContext = containerContext;
  this.leafVertex = leafVertex;
}
 

private Set<String> resolveRacks(List<String> nodes) {
  Set<String> racks = new HashSet<String>();    
  if (nodes != null) {
    for (String node : nodes) {
      // Ensure node requests are accompanied by requests for
      // corresponding rack
      String rack = RackResolver.resolve(node).getNetworkLocation();
      if (rack == null) {
        LOG.warn("Failed to resolve rack for node " + node + ".");
      } else {
        racks.add(rack);
      }
    }
  }
  
  return racks;
}
 

public String resolve(String host) {
  if (hostRackMap.containsKey(host)) {
    return hostRackMap.get(host);
  } else {
    String rack = RackResolver.resolve(host).getNetworkLocation();
    hostRackMap.put(host, rack);
    return rack;
  }
}
 

private void addLeafTask(TaskAttemptToSchedulerEvent event) {
  TaskAttempt taskAttempt = event.getTaskAttempt();
  List<DataLocation> locations = taskAttempt.getTask().getDataLocations();

  for (DataLocation location : locations) {
    String host = location.getHost();
    leafTaskHosts.add(host);

    HostVolumeMapping hostVolumeMapping = leafTaskHostMapping.get(host);
    if (hostVolumeMapping == null) {
      String rack = RackResolver.resolve(host).getNetworkLocation();
      hostVolumeMapping = new HostVolumeMapping(host, rack);
      leafTaskHostMapping.put(host, hostVolumeMapping);
    }
    hostVolumeMapping.addTaskAttempt(location.getVolumeId(), taskAttempt);

    if (LOG.isDebugEnabled()) {
      LOG.debug("Added attempt req to host " + host);
    }

    HashSet<TaskAttemptId> list = leafTasksRackMapping.get(hostVolumeMapping.getRack());
    if (list == null) {
      list = new HashSet<>();
      leafTasksRackMapping.put(hostVolumeMapping.getRack(), list);
    }

    list.add(taskAttempt.getId());

    if (LOG.isDebugEnabled()) {
      LOG.debug("Added attempt req to rack " + hostVolumeMapping.getRack());
    }
  }

  leafTasks.add(taskAttempt.getId());
}
 

@Override
protected void serviceInit(Configuration conf) throws Exception {
  resourceTrackerAddress = conf.getSocketAddr(
      YarnConfiguration.RM_BIND_HOST,
      YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);

  RackResolver.init(conf);
  nextHeartBeatInterval =
      conf.getLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS,
          YarnConfiguration.DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS);
  if (nextHeartBeatInterval <= 0) {
    throw new YarnRuntimeException("Invalid Configuration. "
        + YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS
        + " should be larger than 0.");
  }

  minAllocMb = conf.getInt(
  	YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
  	YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
  minAllocVcores = conf.getInt(
  	YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
  	YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
  minAllocGcores = conf.getInt(
  	YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_GCORES,
  	YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_GCORES);

  minimumNodeManagerVersion = conf.get(
      YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION,
      YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION);

  super.serviceInit(conf);
}
 

@Override
public void removeFragment(FragmentPair fragmentPair) {
  boolean removed = false;
  for (String eachHost : fragmentPair.getLeftFragment().getHosts()) {
    String normalizedHost = NetUtils.normalizeHost(eachHost);
    Map<Integer, FragmentsPerDisk> diskFragmentMap = fragmentHostMapping.get(normalizedHost);
    for (Entry<Integer, FragmentsPerDisk> entry : diskFragmentMap.entrySet()) {
      FragmentsPerDisk fragmentsPerDisk = entry.getValue();
      removed = fragmentsPerDisk.removeFragmentPair(fragmentPair);
      if (removed) {
        if (fragmentsPerDisk.size() == 0) {
          diskFragmentMap.remove(entry.getKey());
        }
        if (diskFragmentMap.size() == 0) {
          fragmentHostMapping.remove(normalizedHost);
        }
        break;
      }
    }
    String rack = RackResolver.resolve(normalizedHost).getNetworkLocation();
    if (rackFragmentMapping.containsKey(rack)) {
      Set<FragmentPair> fragmentPairs = rackFragmentMapping.get(rack);
      fragmentPairs.remove(fragmentPair);
      if (fragmentPairs.size() == 0) {
        rackFragmentMapping.remove(rack);
      }
    }
  }
  if (removed) {
    fragmentNum--;
  }
}
 

/**
 * Randomly select a fragment among the fragments stored on nodes of the same rack with the host.
 * @param host
 * @return a randomly selected fragment
 */
@Override
public FragmentPair getRackLocalFragment(String host) {
  String rack = RackResolver.resolve(host).getNetworkLocation();
  if (rackFragmentMapping.containsKey(rack)) {
    Set<FragmentPair> fragmentPairs = rackFragmentMapping.get(rack);
    if (!fragmentPairs.isEmpty()) {
      return fragmentPairs.iterator().next();
    }
  }
  return null;
}
 

@Override
public void init(Configuration conf) {
  this.systemConf = (TajoConf)conf;
  RackResolver.init(systemConf);
  Runtime.getRuntime().addShutdownHook(new Thread(new ShutdownHook()));
  super.init(conf);
}
 

private void addLeafTask(QueryUnitAttemptScheduleEvent event) {
  QueryUnitAttempt queryUnitAttempt = event.getQueryUnitAttempt();
  List<DataLocation> locations = queryUnitAttempt.getQueryUnit().getDataLocations();

  for (DataLocation location : locations) {
    String host = location.getHost();

    HostVolumeMapping hostVolumeMapping = leafTaskHostMapping.get(host);
    if (hostVolumeMapping == null) {
      String rack = RackResolver.resolve(host).getNetworkLocation();
      hostVolumeMapping = new HostVolumeMapping(host, rack);
      leafTaskHostMapping.put(host, hostVolumeMapping);
    }
    hostVolumeMapping.addQueryUnitAttempt(location.getVolumeId(), queryUnitAttempt);

    if (LOG.isDebugEnabled()) {
      LOG.debug("Added attempt req to host " + host);
    }

    LinkedList<QueryUnitAttemptId> list = leafTasksRackMapping.get(hostVolumeMapping.getRack());
    if (list == null) {
      list = new LinkedList<QueryUnitAttemptId>();
      leafTasksRackMapping.put(hostVolumeMapping.getRack(), list);
    }

    if(!list.contains(queryUnitAttempt.getId())) list.add(queryUnitAttempt.getId());

    if (LOG.isDebugEnabled()) {
      LOG.debug("Added attempt req to rack " + hostVolumeMapping.getRack());
    }
  }

  leafTasks.add(queryUnitAttempt.getId());
}
 

private void addFragment(String host, Integer diskId, FragmentPair fragmentPair) {
  // update the fragment maps per host
  String normalizeHost = NetUtils.normalizeHost(host);
  Map<Integer, FragmentsPerDisk> diskFragmentMap;
  if (fragmentHostMapping.containsKey(normalizeHost)) {
    diskFragmentMap = fragmentHostMapping.get(normalizeHost);
  } else {
    diskFragmentMap = new HashMap<Integer, FragmentsPerDisk>();
    fragmentHostMapping.put(normalizeHost, diskFragmentMap);
  }
  FragmentsPerDisk fragmentsPerDisk;
  if (diskFragmentMap.containsKey(diskId)) {
    fragmentsPerDisk = diskFragmentMap.get(diskId);
  } else {
    fragmentsPerDisk = new FragmentsPerDisk(diskId);
    diskFragmentMap.put(diskId, fragmentsPerDisk);
  }
  fragmentsPerDisk.addFragmentPair(fragmentPair);

  // update the fragment maps per rack
  String rack = RackResolver.resolve(normalizeHost).getNetworkLocation();
  Set<FragmentPair> fragmentPairList;
  if (rackFragmentMapping.containsKey(rack)) {
    fragmentPairList = rackFragmentMapping.get(rack);
  } else {
    fragmentPairList = Collections.newSetFromMap(new HashMap<FragmentPair, Boolean>());
    rackFragmentMapping.put(rack, fragmentPairList);
  }
  fragmentPairList.add(fragmentPair);
}
 

@Override
public CookieContainerRequest assignNewContainer(Container container) {
  String location = RackResolver.resolve(container.getNodeId().getHost())
      .getNetworkLocation();
  CookieContainerRequest assigned = getMatchingRequestWithPriority(container,
      location);
  doBookKeepingForAssignedContainer(assigned, container, location, false);
  return assigned;
}
 

HeldContainer(Container container,
    long nextScheduleTime,
    long containerExpiryTime,
    CookieContainerRequest firstTaskInfo) {
  this.container = container;
  this.nextScheduleTime = nextScheduleTime;
  if (firstTaskInfo != null) {
    this.lastTaskInfo = firstTaskInfo;
    this.firstContainerSignature = firstTaskInfo.getCookie().getContainerSignature();
  }
  this.localityMatchLevel = LocalityMatchLevel.NODE;
  this.containerExpiryTime = containerExpiryTime;
  this.rack = RackResolver.resolve(container.getNodeId().getHost())
      .getNetworkLocation();
}
 

@Override
public CookieContainerRequest assignNewContainer(Container container) {
  String location = RackResolver.resolve(container.getNodeId().getHost())
      .getNetworkLocation();
  CookieContainerRequest assigned = getMatchingRequestWithPriority(container,
      location);
  doBookKeepingForAssignedContainer(assigned, container, location, false);
  return assigned;
}
 

@SuppressWarnings("unchecked")
@Before
public void setup() {
  amrmClient = new TezAMRMClientAsync(new AMRMClientImpl(),
    1000, mock(AMRMClientAsync.CallbackHandler.class));
  RackResolver.init(new Configuration());
}
 

@Override
 protected void serviceInit(Configuration conf) throws Exception {
   
super.serviceInit(conf);
   
   //this.reduceScheduler.serviceInit(conf);
   
   reduceSlowStart = conf.getFloat(
        MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 
       DEFAULT_COMPLETED_MAPS_PERCENT_FOR_REDUCE_SLOWSTART);
   maxReduceRampupLimit = conf.getFloat(
       MRJobConfig.MR_AM_JOB_REDUCE_RAMPUP_UP_LIMIT, 
       MRJobConfig.DEFAULT_MR_AM_JOB_REDUCE_RAMP_UP_LIMIT);
   maxReducePreemptionLimit = conf.getFloat(
       MRJobConfig.MR_AM_JOB_REDUCE_PREEMPTION_LIMIT,
       MRJobConfig.DEFAULT_MR_AM_JOB_REDUCE_PREEMPTION_LIMIT);
   allocationDelayThresholdMs = conf.getInt(
       MRJobConfig.MR_JOB_REDUCER_PREEMPT_DELAY_SEC,
       MRJobConfig.DEFAULT_MR_JOB_REDUCER_PREEMPT_DELAY_SEC) * 1000;//sec -> ms
   maxRunningMaps = conf.getInt(MRJobConfig.JOB_RUNNING_MAP_LIMIT,
       MRJobConfig.DEFAULT_JOB_RUNNING_MAP_LIMIT);
   maxRunningReduces = conf.getInt(MRJobConfig.JOB_RUNNING_REDUCE_LIMIT,
       MRJobConfig.DEFAULT_JOB_RUNNING_REDUCE_LIMIT);
   RackResolver.init(conf);
   retryInterval = getConfig().getLong(MRJobConfig.MR_AM_TO_RM_WAIT_INTERVAL_MS,
                               MRJobConfig.DEFAULT_MR_AM_TO_RM_WAIT_INTERVAL_MS);
   // Init startTime to current time. If all goes well, it will be reset after
   // first attempt to contact RM.
   retrystartTime = System.currentTimeMillis();
 }
 

@Override
protected void serviceInit(Configuration conf) throws Exception {
  resourceTrackerAddress = conf.getSocketAddr(
      YarnConfiguration.RM_BIND_HOST,
      YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);

  RackResolver.init(conf);
  nextHeartBeatInterval =
      conf.getLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS,
          YarnConfiguration.DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS);
  if (nextHeartBeatInterval <= 0) {
    throw new YarnRuntimeException("Invalid Configuration. "
        + YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS
        + " should be larger than 0.");
  }

  minAllocMb = conf.getInt(
  	YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
  	YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
  minAllocVcores = conf.getInt(
  	YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
  	YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);

  minimumNodeManagerVersion = conf.get(
      YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION,
      YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION);

  super.serviceInit(conf);
}
 

@Override
protected void serviceInit(Configuration conf) throws Exception {
  super.serviceInit(conf);
  reduceSlowStart = conf.getFloat(
      MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 
      DEFAULT_COMPLETED_MAPS_PERCENT_FOR_REDUCE_SLOWSTART);
  maxReduceRampupLimit = conf.getFloat(
      MRJobConfig.MR_AM_JOB_REDUCE_RAMPUP_UP_LIMIT, 
      MRJobConfig.DEFAULT_MR_AM_JOB_REDUCE_RAMP_UP_LIMIT);
  maxReducePreemptionLimit = conf.getFloat(
      MRJobConfig.MR_AM_JOB_REDUCE_PREEMPTION_LIMIT,
      MRJobConfig.DEFAULT_MR_AM_JOB_REDUCE_PREEMPTION_LIMIT);
  allocationDelayThresholdMs = conf.getInt(
      MRJobConfig.MR_JOB_REDUCER_PREEMPT_DELAY_SEC,
      MRJobConfig.DEFAULT_MR_JOB_REDUCER_PREEMPT_DELAY_SEC) * 1000;//sec -> ms
  maxRunningMaps = conf.getInt(MRJobConfig.JOB_RUNNING_MAP_LIMIT,
      MRJobConfig.DEFAULT_JOB_RUNNING_MAP_LIMIT);
  maxRunningReduces = conf.getInt(MRJobConfig.JOB_RUNNING_REDUCE_LIMIT,
      MRJobConfig.DEFAULT_JOB_RUNNING_REDUCE_LIMIT);
  RackResolver.init(conf);
  retryInterval = getConfig().getLong(MRJobConfig.MR_AM_TO_RM_WAIT_INTERVAL_MS,
                              MRJobConfig.DEFAULT_MR_AM_TO_RM_WAIT_INTERVAL_MS);
  mapNodeLabelExpression = conf.get(MRJobConfig.MAP_NODE_LABEL_EXP);
  reduceNodeLabelExpression = conf.get(MRJobConfig.REDUCE_NODE_LABEL_EXP);
  // Init startTime to current time. If all goes well, it will be reset after
  // first attempt to contact RM.
  retrystartTime = System.currentTimeMillis();
}
 

/**
 * Simple test some methods of JobHistory
 */
@Test(timeout = 20000)
public void testJobHistoryMethods() throws Exception {
  LOG.info("STARTING testJobHistoryMethods");
  try {
    Configuration configuration = new Configuration();
    configuration
        .setClass(
            NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
            MyResolver.class, DNSToSwitchMapping.class);

    RackResolver.init(configuration);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(configuration);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    app.waitForState(job, JobState.SUCCEEDED);

    JobHistory jobHistory = new JobHistory();
    jobHistory.init(configuration);
    // Method getAllJobs
    Assert.assertEquals(1, jobHistory.getAllJobs().size());
    // and with ApplicationId
    Assert.assertEquals(1, jobHistory.getAllJobs(app.getAppID()).size());

    JobsInfo jobsinfo = jobHistory.getPartialJobs(0L, 10L, null, "default",
        0L, System.currentTimeMillis() + 1, 0L,
        System.currentTimeMillis() + 1, JobState.SUCCEEDED);

    Assert.assertEquals(1, jobsinfo.getJobs().size());
    Assert.assertNotNull(jobHistory.getApplicationAttemptId());
    // test Application Id
    Assert.assertEquals("application_0_0000", jobHistory.getApplicationID()
        .toString());
    Assert
        .assertEquals("Job History Server", jobHistory.getApplicationName());
    // method does not work
    Assert.assertNull(jobHistory.getEventHandler());
    // method does not work
    Assert.assertNull(jobHistory.getClock());
    // method does not work
    Assert.assertNull(jobHistory.getClusterInfo());

  } finally {
    LOG.info("FINISHED testJobHistoryMethods");
  }
}
 

HeldContainer(Container container) {
  this.container = container;
  this.rack = RackResolver.resolve(container.getNodeId().getHost()).getNetworkLocation();
}
 

@Override
protected void serviceInit(Configuration conf) throws Exception {
  RackResolver.init(conf);
  super.serviceInit(conf);
}
 

public TaskAttemptImpl(TaskId taskId, int i, 
    EventHandler eventHandler,
    TaskAttemptListener taskAttemptListener, Path jobFile, int partition,
    JobConf conf, String[] dataLocalHosts,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    AppContext appContext) {
  oldJobId = TypeConverter.fromYarn(taskId.getJobId());
  this.conf = conf;
  this.clock = clock;
  attemptId = recordFactory.newRecordInstance(TaskAttemptId.class);
  attemptId.setTaskId(taskId);
  attemptId.setId(i);
  this.taskAttemptListener = taskAttemptListener;
  this.appContext = appContext;

  // Initialize reportedStatus
  reportedStatus = new TaskAttemptStatus();
  initTaskAttemptStatus(reportedStatus);

  ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
  readLock = readWriteLock.readLock();
  writeLock = readWriteLock.writeLock();

  this.credentials = credentials;
  this.jobToken = jobToken;
  this.eventHandler = eventHandler;
  this.jobFile = jobFile;
  this.partition = partition;

  //TODO:create the resource reqt for this Task attempt
  this.resourceCapability = recordFactory.newRecordInstance(Resource.class);
  this.resourceCapability.setMemory(
      getMemoryRequired(conf, taskId.getTaskType()));
  this.resourceCapability.setVirtualCores(
      getCpuRequired(conf, taskId.getTaskType()));
  this.resourceCapability.setGpuCores(
      getGpuRequired(conf, taskId.getTaskType()));

  this.dataLocalHosts = resolveHosts(dataLocalHosts);
  RackResolver.init(conf);
  this.dataLocalRacks = new HashSet<String>(); 
  for (String host : this.dataLocalHosts) {
    this.dataLocalRacks.add(RackResolver.resolve(host).getNetworkLocation());
  }

  locality = Locality.OFF_SWITCH;
  avataar = Avataar.VIRGIN;

  // This "this leak" is okay because the retained pointer is in an
  //  instance variable.
  stateMachine = stateMachineFactory.make(this);
}
 

@Override
public void transition(TaskAttemptImpl ta, TaskAttemptEvent origEvent) {
  TaskAttemptEventSubmitted event = (TaskAttemptEventSubmitted) origEvent;

  AMContainer amContainer = ta.appContext.getAllContainers().get(event.getContainerId());
  Container container = amContainer.getContainer();

  ta.allocationTime = amContainer.getCurrentTaskAttemptAllocationTime();
  ta.container = container;
  ta.containerId = event.getContainerId();
  ta.containerNodeId = container.getNodeId();
  ta.nodeHttpAddress = StringInterner.weakIntern(container.getNodeHttpAddress());
  ta.nodeRackName = StringInterner.weakIntern(RackResolver.resolve(ta.containerNodeId.getHost())
      .getNetworkLocation());
  ta.lastNotifyProgressTimestamp = ta.clock.getTime();

  ta.setLaunchTime();

  // TODO Resolve to host / IP in case of a local address.
  InetSocketAddress nodeHttpInetAddr = NetUtils
      .createSocketAddr(ta.nodeHttpAddress); // TODO: Costly?
  ta.trackerName = StringInterner.weakIntern(nodeHttpInetAddr.getHostName());
  ta.httpPort = nodeHttpInetAddr.getPort();
  ta.sendEvent(createDAGCounterUpdateEventTALaunched(ta));

  LOG.info("TaskAttempt: [" + ta.attemptId + "] submitted."
      + " Is using containerId: [" + ta.containerId + "]" + " on NM: ["
      + ta.containerNodeId + "]");

  // JobHistoryEvent.
  // The started event represents when the attempt was submitted to the executor.
  ta.logJobHistoryAttemptStarted();

  // TODO Remove after HDFS-5098
  // Compute LOCALITY counter for this task.
  if (ta.taskHosts.contains(ta.containerNodeId.getHost())) {
    ta.localityCounter = DAGCounter.DATA_LOCAL_TASKS;
  } else if (ta.taskRacks.contains(ta.nodeRackName)) {
    ta.localityCounter = DAGCounter.RACK_LOCAL_TASKS;
  } else {
    // Not computing this if the task does not have locality information.
    if (ta.getTaskLocationHint() != null) {
      ta.localityCounter = DAGCounter.OTHER_LOCAL_TASKS;
    }
  }

  // Inform the Task
  ta.sendEvent(new TaskEventTALaunched(ta.attemptId));

  if (ta.isSpeculationEnabled()) {
    ta.sendEvent(new SpeculatorEventTaskAttemptStatusUpdate(ta.attemptId, TaskAttemptState.RUNNING,
        ta.launchTime, true));
  }

  ta.sendEvent(
      new AMSchedulerEventTAStateUpdated(ta, TaskScheduler.SchedulerTaskState.SUBMITTED,
          ta.getVertex().getTaskSchedulerIdentifier()));
  ta.taskHeartbeatHandler.register(ta.attemptId);
}
 

@SuppressWarnings("rawtypes")
public TaskAttemptImpl(TezTaskAttemptID attemptId, EventHandler eventHandler,
    TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Configuration conf, Clock clock,
    TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext,
    boolean isRescheduled,
    Resource resource, ContainerContext containerContext, boolean leafVertex,
    Task task, TaskLocationHint locationHint, TaskSpec taskSpec,
    TezTaskAttemptID schedulingCausalTA) {

  ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
  this.readLock = rwLock.readLock();
  this.writeLock = rwLock.writeLock();
  this.attemptId = attemptId;
  this.eventHandler = eventHandler;
  //Reported status
  this.conf = conf;
  this.clock = clock;
  this.taskHeartbeatHandler = taskHeartbeatHandler;
  this.appContext = appContext;
  this.vertex = task.getVertex();
  this.task = task;
  this.locationHint = locationHint;
  this.taskSpec = taskSpec;
  this.creationCausalTA = schedulingCausalTA;
  this.creationTime = clock.getTime();

  this.reportedStatus = new TaskAttemptStatus(this.attemptId);
  initTaskAttemptStatus(reportedStatus);
  RackResolver.init(conf);
  this.stateMachine = stateMachineFactory.make(this);
  this.isRescheduled = isRescheduled;
  this.taskResource = resource;
  this.containerContext = containerContext;
  this.leafVertex = leafVertex;
  this.hungIntervalMax = conf.getLong(
      TezConfiguration.TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS, 
      TezConfiguration.TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS_DEFAULT);

  this.recoveryData = appContext.getDAGRecoveryData() == null ?
      null : appContext.getDAGRecoveryData().getTaskAttemptRecoveryData(attemptId);
}
 

public static void initializeMockRackResolver() {
  Configuration rackResolverConf = new Configuration(false);
  rackResolverConf.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MockDNSToSwitchMapping.class.getName());
  RackResolver.init(rackResolverConf);
}
 

@SuppressWarnings("unchecked")
@Test(timeout=5000)
public void testTaskSchedulerRandomReuseExpireTime() throws Exception {
  RackResolver.init(new YarnConfiguration());
  TaskSchedulerAppCallback mockApp = mock(TaskSchedulerAppCallback.class);
  AppContext mockAppContext = mock(AppContext.class);
  when(mockAppContext.getAMState()).thenReturn(DAGAppMasterState.RUNNING);

  TezAMRMClientAsync<CookieContainerRequest> mockRMClient =
                                                mock(TezAMRMClientAsync.class);

  String appHost = "host";
  int appPort = 0;
  String appUrl = "url";
  TaskSchedulerWithDrainableAppCallback scheduler1 =
    new TaskSchedulerWithDrainableAppCallback(
      mockApp, new AlwaysMatchesContainerMatcher(), appHost, appPort,
      appUrl, mockRMClient, mockAppContext);
  TaskSchedulerWithDrainableAppCallback scheduler2 =
      new TaskSchedulerWithDrainableAppCallback(
        mockApp, new AlwaysMatchesContainerMatcher(), appHost, appPort,
        appUrl, mockRMClient, mockAppContext);

  long minTime = 1000l;
  long maxTime = 100000l;
  Configuration conf1 = new Configuration();
  conf1.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, minTime);
  conf1.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, minTime);
  scheduler1.init(conf1);
  Configuration conf2 = new Configuration();
  conf2.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, minTime);
  conf2.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, maxTime);
  scheduler2.init(conf2);

  RegisterApplicationMasterResponse mockRegResponse =
                              mock(RegisterApplicationMasterResponse.class);
  Resource mockMaxResource = mock(Resource.class);
  Map<ApplicationAccessType, String> mockAcls = mock(Map.class);
  when(mockRegResponse.getMaximumResourceCapability()).
                                                 thenReturn(mockMaxResource);
  when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls);
  when(mockRMClient.
        registerApplicationMaster(anyString(), anyInt(), anyString())).
                                                 thenReturn(mockRegResponse);
  Resource mockClusterResource = mock(Resource.class);
  when(mockRMClient.getAvailableResources()).
                                            thenReturn(mockClusterResource);

  scheduler1.start();
  scheduler2.start();
  
  // when min == max the expire time is always min
  for (int i=0; i<10; ++i) {
    Assert.assertEquals(minTime, scheduler1.getHeldContainerExpireTime(0));
  }
  
  long lastExpireTime = 0;
  // when min < max the expire time is random in between min and max
  for (int i=0; i<10; ++i) {
    long currExpireTime = scheduler2.getHeldContainerExpireTime(0);
    Assert.assertTrue(
        "min: " + minTime + " curr: " + currExpireTime + " max: " + maxTime,
        (minTime <= currExpireTime && currExpireTime <= maxTime));
    Assert.assertNotEquals(lastExpireTime, currExpireTime);
    lastExpireTime = currExpireTime;
  }

  String appMsg = "success";
  AppFinalStatus finalStatus =
      new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl);
  when(mockApp.getFinalAppStatus()).thenReturn(finalStatus);
  scheduler1.stop();
  scheduler1.close();
  scheduler2.stop();
  scheduler2.close();
}
 

@Before
public void setup() {
  amrmClient = new TezAMRMClientAsync(new AMRMClientImpl(),
    1000, mock(AMRMClientAsync.CallbackHandler.class));
  RackResolver.init(new Configuration());
}