Java Code Examples for org.apache.hadoop.yarn.util.RackResolver#init()

@Override
protected void serviceInit(Configuration conf) throws Exception {
  resourceTrackerAddress = conf.getSocketAddr(
      YarnConfiguration.RM_BIND_HOST,
      YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);

  RackResolver.init(conf);
  nextHeartBeatInterval =
      conf.getLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS,
          YarnConfiguration.DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS);
  if (nextHeartBeatInterval <= 0) {
    throw new YarnRuntimeException("Invalid Configuration. "
        + YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS
        + " should be larger than 0.");
  }

  minAllocMb = conf.getInt(
  	YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
  	YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
  minAllocVcores = conf.getInt(
  	YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
  	YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
  minAllocGcores = conf.getInt(
  	YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_GCORES,
  	YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_GCORES);

  minimumNodeManagerVersion = conf.get(
      YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION,
      YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION);

  super.serviceInit(conf);
}
 

@SuppressWarnings("unchecked")
@Before
public void setup() {
  amrmClient = new TezAMRMClientAsync(new AMRMClientImpl(),
    1000, mock(AMRMClientAsync.CallbackHandler.class));
  RackResolver.init(new Configuration());
}
 

@Override
public void init(Configuration conf) {
  this.systemConf = (TajoConf)conf;
  RackResolver.init(systemConf);
  Runtime.getRuntime().addShutdownHook(new Thread(new ShutdownHook()));
  super.init(conf);
}
 

@Override
protected void serviceInit(Configuration conf) throws Exception {
  resourceTrackerAddress = conf.getSocketAddr(
      YarnConfiguration.RM_BIND_HOST,
      YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);

  RackResolver.init(conf);
  nextHeartBeatInterval =
      conf.getLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS,
          YarnConfiguration.DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS);
  if (nextHeartBeatInterval <= 0) {
    throw new YarnRuntimeException("Invalid Configuration. "
        + YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS
        + " should be larger than 0.");
  }

  minAllocMb = conf.getInt(
  	YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
  	YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
  minAllocVcores = conf.getInt(
  	YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
  	YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);

  minimumNodeManagerVersion = conf.get(
      YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION,
      YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION);

  super.serviceInit(conf);
}
 

@Override
 protected void serviceInit(Configuration conf) throws Exception {
   
super.serviceInit(conf);
   
   //this.reduceScheduler.serviceInit(conf);
   
   reduceSlowStart = conf.getFloat(
        MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 
       DEFAULT_COMPLETED_MAPS_PERCENT_FOR_REDUCE_SLOWSTART);
   maxReduceRampupLimit = conf.getFloat(
       MRJobConfig.MR_AM_JOB_REDUCE_RAMPUP_UP_LIMIT, 
       MRJobConfig.DEFAULT_MR_AM_JOB_REDUCE_RAMP_UP_LIMIT);
   maxReducePreemptionLimit = conf.getFloat(
       MRJobConfig.MR_AM_JOB_REDUCE_PREEMPTION_LIMIT,
       MRJobConfig.DEFAULT_MR_AM_JOB_REDUCE_PREEMPTION_LIMIT);
   allocationDelayThresholdMs = conf.getInt(
       MRJobConfig.MR_JOB_REDUCER_PREEMPT_DELAY_SEC,
       MRJobConfig.DEFAULT_MR_JOB_REDUCER_PREEMPT_DELAY_SEC) * 1000;//sec -> ms
   maxRunningMaps = conf.getInt(MRJobConfig.JOB_RUNNING_MAP_LIMIT,
       MRJobConfig.DEFAULT_JOB_RUNNING_MAP_LIMIT);
   maxRunningReduces = conf.getInt(MRJobConfig.JOB_RUNNING_REDUCE_LIMIT,
       MRJobConfig.DEFAULT_JOB_RUNNING_REDUCE_LIMIT);
   RackResolver.init(conf);
   retryInterval = getConfig().getLong(MRJobConfig.MR_AM_TO_RM_WAIT_INTERVAL_MS,
                               MRJobConfig.DEFAULT_MR_AM_TO_RM_WAIT_INTERVAL_MS);
   // Init startTime to current time. If all goes well, it will be reset after
   // first attempt to contact RM.
   retrystartTime = System.currentTimeMillis();
 }
 

@Before
public void setup() {
  amrmClient = new TezAMRMClientAsync(new AMRMClientImpl(),
    1000, mock(AMRMClientAsync.CallbackHandler.class));
  RackResolver.init(new Configuration());
}
 

@Override
public void init(Configuration conf) {
  Runtime.getRuntime().addShutdownHook(new Thread(new ShutdownHook()));

  this.systemConf = (TajoConf)conf;
  RackResolver.init(systemConf);

  this.connPool = RpcConnectionPool.getPool(systemConf);
  this.workerContext = new WorkerContext();

  String resourceManagerClassName = systemConf.getVar(ConfVars.RESOURCE_MANAGER_CLASS);

  boolean randomPort = true;
  if(resourceManagerClassName.indexOf(TajoWorkerResourceManager.class.getName()) >= 0) {
    randomPort = false;
  }
  int clientPort = systemConf.getSocketAddrVar(ConfVars.WORKER_CLIENT_RPC_ADDRESS).getPort();
  int peerRpcPort = systemConf.getSocketAddrVar(ConfVars.WORKER_PEER_RPC_ADDRESS).getPort();
  int qmManagerPort = systemConf.getSocketAddrVar(ConfVars.WORKER_QM_RPC_ADDRESS).getPort();

  if(randomPort) {
    clientPort = 0;
    peerRpcPort = 0;
    qmManagerPort = 0;
    systemConf.setIntVar(ConfVars.PULLSERVER_PORT, 0);
  }

  // querymaster worker
  tajoWorkerClientService = new TajoWorkerClientService(workerContext, clientPort);
  addService(tajoWorkerClientService);

  queryMasterManagerService = new QueryMasterManagerService(workerContext, qmManagerPort);
  addService(queryMasterManagerService);

  // taskrunner worker
  taskRunnerManager = new TaskRunnerManager(workerContext);
  addService(taskRunnerManager);

  tajoWorkerManagerService = new TajoWorkerManagerService(workerContext, peerRpcPort);
  addService(tajoWorkerManagerService);

  if(!yarnContainerMode) {
    if(taskRunnerMode) {
      pullService = new TajoPullServerService();
      addService(pullService);
    }

    if (!systemConf.get(CommonTestingUtil.TAJO_TEST, "FALSE").equalsIgnoreCase("TRUE")) {
      try {
        httpPort = systemConf.getSocketAddrVar(ConfVars.WORKER_INFO_ADDRESS).getPort();
        if(queryMasterMode && !taskRunnerMode) {
          //If QueryMaster and TaskRunner run on single host, http port conflicts
          httpPort = systemConf.getSocketAddrVar(ConfVars.WORKER_QM_INFO_ADDRESS).getPort();
        }
        webServer = StaticHttpServer.getInstance(this ,"worker", null, httpPort ,
            true, null, systemConf, null);
        webServer.start();
        httpPort = webServer.getPort();
        LOG.info("Worker info server started:" + httpPort);

        deletionService = new DeletionService(getMountPath().size(), 0);
        if(systemConf.getBoolVar(ConfVars.WORKER_TEMPORAL_DIR_CLEANUP)){
          getWorkerContext().cleanupTemporalDirectories();
        }
      } catch (IOException e) {
        LOG.error(e.getMessage(), e);
      }
    }
  }

  LOG.info("Tajo Worker started: queryMaster=" + queryMasterMode + " taskRunner=" + taskRunnerMode +
      ", qmRpcPort=" + qmManagerPort +
      ",yarnContainer=" + yarnContainerMode + ", clientPort=" + clientPort +
      ", peerRpcPort=" + peerRpcPort + ":" + qmManagerPort + ",httpPort" + httpPort);

  super.init(conf);

  if(yarnContainerMode && queryMasterMode) {
    tajoMasterAddress = NetUtils.createSocketAddr(cmdArgs[2]);
    connectToCatalog();

    QueryId queryId = TajoIdUtils.parseQueryId(cmdArgs[1]);
    queryMasterManagerService.getQueryMaster().reportQueryStatusToQueryMaster(
        queryId, TajoProtos.QueryState.QUERY_MASTER_LAUNCHED);
  } else if(yarnContainerMode && taskRunnerMode) { //TaskRunner mode
    taskRunnerManager.startTask(cmdArgs);
  } else {
    tajoMasterAddress = NetUtils.createSocketAddr(systemConf.getVar(ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS));
    workerResourceTrackerAddr = NetUtils.createSocketAddr(systemConf.getVar(ConfVars.RESOURCE_TRACKER_RPC_ADDRESS));
    connectToCatalog();
  }

  workerHeartbeatThread = new WorkerHeartbeatService(workerContext);
  workerHeartbeatThread.init(conf);
  addIfService(workerHeartbeatThread);
}
 

public static void initializeMockRackResolver() {
  Configuration rackResolverConf = new Configuration(false);
  rackResolverConf.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MockDNSToSwitchMapping.class.getName());
  RackResolver.init(rackResolverConf);
}
 

@Override
public void serviceInit(Configuration conf) throws Exception {

  this.systemConf = TUtil.checkTypeAndGet(conf, TajoConf.class);
  ShutdownHookManager.get().addShutdownHook(new ShutdownHook(), SHUTDOWN_HOOK_PRIORITY);

  context = new MasterContext(systemConf);
  clock = new SystemClock();
  RackResolver.init(systemConf);

  initResourceManager();

  this.dispatcher = new AsyncDispatcher();
  addIfService(dispatcher);

  // check the system directory and create if they are not created.
  checkAndInitializeSystemDirectories();
  diagnoseTajoMaster();

  catalogServer = new CatalogServer(TablespaceManager.getMetadataProviders(), loadFunctions());
  addIfService(catalogServer);
  catalog = new LocalCatalogWrapper(catalogServer, systemConf);

  sessionManager = new SessionManager(dispatcher);
  addIfService(sessionManager);

  globalEngine = new GlobalEngine(context);
  addIfService(globalEngine);

  queryManager = new QueryManager(context);
  addIfService(queryManager);

  tajoMasterClientService = new TajoMasterClientService(context);
  addIfService(tajoMasterClientService);

  asyncTaskService = new AsyncTaskService(context);
  addIfService(asyncTaskService);

  tajoMasterService = new QueryCoordinatorService(context);
  addIfService(tajoMasterService);

  restServer = new TajoRestService(context);
  addIfService(restServer);

  PythonScriptEngine.initPythonScriptEngineFiles();

  // Try to start up all services in TajoMaster.
  // If anyone is failed, the master prints out the errors and immediately should shutdowns
  try {
    super.serviceInit(systemConf);
  } catch (Throwable t) {
    t.printStackTrace();
    Runtime.getRuntime().halt(-1);
  }
  LOG.info("Tajo Master is initialized.");
}
 

/**
 * Simple test some methods of JobHistory
 */
@Test(timeout = 20000)
public void testJobHistoryMethods() throws Exception {
  LOG.info("STARTING testJobHistoryMethods");
  try {
    Configuration configuration = new Configuration();
    configuration
        .setClass(
            NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
            MyResolver.class, DNSToSwitchMapping.class);

    RackResolver.init(configuration);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(configuration);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    app.waitForState(job, JobState.SUCCEEDED);

    JobHistory jobHistory = new JobHistory();
    jobHistory.init(configuration);
    // Method getAllJobs
    Assert.assertEquals(1, jobHistory.getAllJobs().size());
    // and with ApplicationId
    Assert.assertEquals(1, jobHistory.getAllJobs(app.getAppID()).size());

    JobsInfo jobsinfo = jobHistory.getPartialJobs(0L, 10L, null, "default",
        0L, System.currentTimeMillis() + 1, 0L,
        System.currentTimeMillis() + 1, JobState.SUCCEEDED);

    Assert.assertEquals(1, jobsinfo.getJobs().size());
    Assert.assertNotNull(jobHistory.getApplicationAttemptId());
    // test Application Id
    Assert.assertEquals("application_0_0000", jobHistory.getApplicationID()
        .toString());
    Assert
        .assertEquals("Job History Server", jobHistory.getApplicationName());
    // method does not work
    Assert.assertNull(jobHistory.getEventHandler());
    // method does not work
    Assert.assertNull(jobHistory.getClock());
    // method does not work
    Assert.assertNull(jobHistory.getClusterInfo());

  } finally {
    LOG.info("FINISHED testJobHistoryMethods");
  }
}
 

/**
 * test clean old history files. Files should be deleted after 1 week by
 * default.
 */
@Test(timeout = 15000)
public void testDeleteFileInfo() throws Exception {
  LOG.info("STARTING testDeleteFileInfo");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);

    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();

    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);
    HistoryFileManager hfm = new HistoryFileManager();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    hfm.initExisting();
    // wait for move files form the done_intermediate directory to the gone
    // directory
    while (fileInfo.isMovePending()) {
      Thread.sleep(300);
    }

    Assert.assertNotNull(hfm.jobListCache.values());

    // try to remove fileInfo
    hfm.clean();
    // check that fileInfo does not deleted
    Assert.assertFalse(fileInfo.isDeleted());
    // correct live time
    hfm.setMaxHistoryAge(-1);
    hfm.clean();
    hfm.stop();
    Assert.assertTrue("Thread pool shutdown",
        hfm.moveToDoneExecutor.isTerminated());
    // should be deleted !
    Assert.assertTrue("file should be deleted ", fileInfo.isDeleted());

  } finally {
    LOG.info("FINISHED testDeleteFileInfo");
  }
}
 

@Test(timeout = 50000)
public void testScanningOldDirs() throws Exception {
  LOG.info("STARTING testScanningOldDirs");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);
    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);

    HistoryFileManagerForTest hfm = new HistoryFileManagerForTest();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    Assert.assertNotNull("Unable to locate job history", fileInfo);

    // force the manager to "forget" the job
    hfm.deleteJobFromJobListCache(fileInfo);
    final int msecPerSleep = 10;
    int msecToSleep = 10 * 1000;
    while (fileInfo.isMovePending() && msecToSleep > 0) {
      Assert.assertTrue(!fileInfo.didMoveFail());
      msecToSleep -= msecPerSleep;
      Thread.sleep(msecPerSleep);
    }
    Assert.assertTrue("Timeout waiting for history move", msecToSleep > 0);

    fileInfo = hfm.getFileInfo(jobId);
    hfm.stop();
    Assert.assertNotNull("Unable to locate old job history", fileInfo);
    Assert.assertTrue("HistoryFileManager not shutdown properly",
        hfm.moveToDoneExecutor.isTerminated());
  } finally {
    LOG.info("FINISHED testScanningOldDirs");
  }
}
 

@SuppressWarnings("unchecked")
@Test(timeout=5000)
public void testTaskSchedulerRandomReuseExpireTime() throws Exception {
  RackResolver.init(new YarnConfiguration());
  TaskSchedulerAppCallback mockApp = mock(TaskSchedulerAppCallback.class);
  AppContext mockAppContext = mock(AppContext.class);
  when(mockAppContext.getAMState()).thenReturn(DAGAppMasterState.RUNNING);

  TezAMRMClientAsync<CookieContainerRequest> mockRMClient =
                                                mock(TezAMRMClientAsync.class);

  String appHost = "host";
  int appPort = 0;
  String appUrl = "url";
  TaskSchedulerWithDrainableAppCallback scheduler1 =
    new TaskSchedulerWithDrainableAppCallback(
      mockApp, new AlwaysMatchesContainerMatcher(), appHost, appPort,
      appUrl, mockRMClient, mockAppContext);
  TaskSchedulerWithDrainableAppCallback scheduler2 =
      new TaskSchedulerWithDrainableAppCallback(
        mockApp, new AlwaysMatchesContainerMatcher(), appHost, appPort,
        appUrl, mockRMClient, mockAppContext);

  long minTime = 1000l;
  long maxTime = 100000l;
  Configuration conf1 = new Configuration();
  conf1.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, minTime);
  conf1.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, minTime);
  scheduler1.init(conf1);
  Configuration conf2 = new Configuration();
  conf2.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MIN_MILLIS, minTime);
  conf2.setLong(TezConfiguration.TEZ_AM_CONTAINER_IDLE_RELEASE_TIMEOUT_MAX_MILLIS, maxTime);
  scheduler2.init(conf2);

  RegisterApplicationMasterResponse mockRegResponse =
                              mock(RegisterApplicationMasterResponse.class);
  Resource mockMaxResource = mock(Resource.class);
  Map<ApplicationAccessType, String> mockAcls = mock(Map.class);
  when(mockRegResponse.getMaximumResourceCapability()).
                                                 thenReturn(mockMaxResource);
  when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls);
  when(mockRMClient.
        registerApplicationMaster(anyString(), anyInt(), anyString())).
                                                 thenReturn(mockRegResponse);
  Resource mockClusterResource = mock(Resource.class);
  when(mockRMClient.getAvailableResources()).
                                            thenReturn(mockClusterResource);

  scheduler1.start();
  scheduler2.start();
  
  // when min == max the expire time is always min
  for (int i=0; i<10; ++i) {
    Assert.assertEquals(minTime, scheduler1.getHeldContainerExpireTime(0));
  }
  
  long lastExpireTime = 0;
  // when min < max the expire time is random in between min and max
  for (int i=0; i<10; ++i) {
    long currExpireTime = scheduler2.getHeldContainerExpireTime(0);
    Assert.assertTrue(
        "min: " + minTime + " curr: " + currExpireTime + " max: " + maxTime,
        (minTime <= currExpireTime && currExpireTime <= maxTime));
    Assert.assertNotEquals(lastExpireTime, currExpireTime);
    lastExpireTime = currExpireTime;
  }

  String appMsg = "success";
  AppFinalStatus finalStatus =
      new AppFinalStatus(FinalApplicationStatus.SUCCEEDED, appMsg, appUrl);
  when(mockApp.getFinalAppStatus()).thenReturn(finalStatus);
  scheduler1.stop();
  scheduler1.close();
  scheduler2.stop();
  scheduler2.close();
}
 

@Override
protected void serviceInit(Configuration conf) throws Exception {
  RackResolver.init(conf);
  super.serviceInit(conf);
}
 

/**
 * Simple test some methods of JobHistory
 */
@Test(timeout = 20000)
public void testJobHistoryMethods() throws Exception {
  LOG.info("STARTING testJobHistoryMethods");
  try {
    Configuration configuration = new Configuration();
    configuration
        .setClass(
            NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
            MyResolver.class, DNSToSwitchMapping.class);

    RackResolver.init(configuration);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(configuration);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    app.waitForState(job, JobState.SUCCEEDED);

    JobHistory jobHistory = new JobHistory();
    jobHistory.init(configuration);
    // Method getAllJobs
    Assert.assertEquals(1, jobHistory.getAllJobs().size());
    // and with ApplicationId
    Assert.assertEquals(1, jobHistory.getAllJobs(app.getAppID()).size());

    JobsInfo jobsinfo = jobHistory.getPartialJobs(0L, 10L, null, "default",
        0L, System.currentTimeMillis() + 1, 0L,
        System.currentTimeMillis() + 1, JobState.SUCCEEDED);

    Assert.assertEquals(1, jobsinfo.getJobs().size());
    Assert.assertNotNull(jobHistory.getApplicationAttemptId());
    // test Application Id
    Assert.assertEquals("application_0_0000", jobHistory.getApplicationID()
        .toString());
    Assert
        .assertEquals("Job History Server", jobHistory.getApplicationName());
    // method does not work
    Assert.assertNull(jobHistory.getEventHandler());
    // method does not work
    Assert.assertNull(jobHistory.getClock());
    // method does not work
    Assert.assertNull(jobHistory.getClusterInfo());

  } finally {
    LOG.info("FINISHED testJobHistoryMethods");
  }
}
 

/**
 * test clean old history files. Files should be deleted after 1 week by
 * default.
 */
@Test(timeout = 15000)
public void testDeleteFileInfo() throws Exception {
  LOG.info("STARTING testDeleteFileInfo");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);

    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();

    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);
    HistoryFileManager hfm = new HistoryFileManager();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    hfm.initExisting();
    // wait for move files form the done_intermediate directory to the gone
    // directory
    while (fileInfo.isMovePending()) {
      Thread.sleep(300);
    }

    Assert.assertNotNull(hfm.jobListCache.values());

    // try to remove fileInfo
    hfm.clean();
    // check that fileInfo does not deleted
    Assert.assertFalse(fileInfo.isDeleted());
    // correct live time
    hfm.setMaxHistoryAge(-1);
    hfm.clean();
    hfm.stop();
    Assert.assertTrue("Thread pool shutdown",
        hfm.moveToDoneExecutor.isTerminated());
    // should be deleted !
    Assert.assertTrue("file should be deleted ", fileInfo.isDeleted());

  } finally {
    LOG.info("FINISHED testDeleteFileInfo");
  }
}
 

@Test(timeout = 50000)
public void testScanningOldDirs() throws Exception {
  LOG.info("STARTING testScanningOldDirs");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);
    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);

    HistoryFileManagerForTest hfm = new HistoryFileManagerForTest();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    Assert.assertNotNull("Unable to locate job history", fileInfo);

    // force the manager to "forget" the job
    hfm.deleteJobFromJobListCache(fileInfo);
    final int msecPerSleep = 10;
    int msecToSleep = 10 * 1000;
    while (fileInfo.isMovePending() && msecToSleep > 0) {
      Assert.assertTrue(!fileInfo.didMoveFail());
      msecToSleep -= msecPerSleep;
      Thread.sleep(msecPerSleep);
    }
    Assert.assertTrue("Timeout waiting for history move", msecToSleep > 0);

    fileInfo = hfm.getFileInfo(jobId);
    hfm.stop();
    Assert.assertNotNull("Unable to locate old job history", fileInfo);
    Assert.assertTrue("HistoryFileManager not shutdown properly",
        hfm.moveToDoneExecutor.isTerminated());
  } finally {
    LOG.info("FINISHED testScanningOldDirs");
  }
}
 

@SuppressWarnings("rawtypes")
public TaskAttemptImpl(TezTaskAttemptID attemptId, EventHandler eventHandler,
    TaskCommunicatorManagerInterface taskCommunicatorManagerInterface, Configuration conf, Clock clock,
    TaskHeartbeatHandler taskHeartbeatHandler, AppContext appContext,
    boolean isRescheduled,
    Resource resource, ContainerContext containerContext, boolean leafVertex,
    Task task, TaskLocationHint locationHint, TaskSpec taskSpec,
    TezTaskAttemptID schedulingCausalTA) {

  ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
  this.readLock = rwLock.readLock();
  this.writeLock = rwLock.writeLock();
  this.attemptId = attemptId;
  this.eventHandler = eventHandler;
  //Reported status
  this.conf = conf;
  this.clock = clock;
  this.taskHeartbeatHandler = taskHeartbeatHandler;
  this.appContext = appContext;
  this.vertex = task.getVertex();
  this.task = task;
  this.locationHint = locationHint;
  this.taskSpec = taskSpec;
  this.creationCausalTA = schedulingCausalTA;
  this.creationTime = clock.getTime();

  this.reportedStatus = new TaskAttemptStatus(this.attemptId);
  initTaskAttemptStatus(reportedStatus);
  RackResolver.init(conf);
  this.stateMachine = stateMachineFactory.make(this);
  this.isRescheduled = isRescheduled;
  this.taskResource = resource;
  this.containerContext = containerContext;
  this.leafVertex = leafVertex;
  this.hungIntervalMax = conf.getLong(
      TezConfiguration.TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS, 
      TezConfiguration.TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS_DEFAULT);

  this.recoveryData = appContext.getDAGRecoveryData() == null ?
      null : appContext.getDAGRecoveryData().getTaskAttemptRecoveryData(attemptId);
}
 

public TaskAttemptImpl(TaskId taskId, int i, 
    EventHandler eventHandler,
    TaskAttemptListener taskAttemptListener, Path jobFile, int partition,
    JobConf conf, String[] dataLocalHosts,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    AppContext appContext) {
  oldJobId = TypeConverter.fromYarn(taskId.getJobId());
  this.conf = conf;
  this.clock = clock;
  attemptId = recordFactory.newRecordInstance(TaskAttemptId.class);
  attemptId.setTaskId(taskId);
  attemptId.setId(i);
  this.taskAttemptListener = taskAttemptListener;
  this.appContext = appContext;

  // Initialize reportedStatus
  reportedStatus = new TaskAttemptStatus();
  initTaskAttemptStatus(reportedStatus);

  ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
  readLock = readWriteLock.readLock();
  writeLock = readWriteLock.writeLock();

  this.credentials = credentials;
  this.jobToken = jobToken;
  this.eventHandler = eventHandler;
  this.jobFile = jobFile;
  this.partition = partition;

  //TODO:create the resource reqt for this Task attempt
  this.resourceCapability = recordFactory.newRecordInstance(Resource.class);
  this.resourceCapability.setMemory(
      getMemoryRequired(conf, taskId.getTaskType()));
  this.resourceCapability.setVirtualCores(
      getCpuRequired(conf, taskId.getTaskType()));
  this.resourceCapability.setGpuCores(
      getGpuRequired(conf, taskId.getTaskType()));

  this.dataLocalHosts = resolveHosts(dataLocalHosts);
  RackResolver.init(conf);
  this.dataLocalRacks = new HashSet<String>(); 
  for (String host : this.dataLocalHosts) {
    this.dataLocalRacks.add(RackResolver.resolve(host).getNetworkLocation());
  }

  locality = Locality.OFF_SWITCH;
  avataar = Avataar.VIRGIN;

  // This "this leak" is okay because the retained pointer is in an
  //  instance variable.
  stateMachine = stateMachineFactory.make(this);
}
 

@Override
protected void serviceInit(Configuration conf) throws Exception {
  RackResolver.init(conf);
  super.serviceInit(conf);
}