org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils Java Examples

private void makeDoneSubdir(Path path) throws IOException {
  try {
    doneDirFc.getFileStatus(path);
    existingDoneSubdirs.add(path);
  } catch (FileNotFoundException fnfE) {
    try {
      FsPermission fsp = new FsPermission(
          JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION);
      doneDirFc.mkdir(path, fsp, true);
      FileStatus fsStatus = doneDirFc.getFileStatus(path);
      LOG.info("Perms after creating " + fsStatus.getPermission().toShort()
          + ", Expected: " + fsp.toShort());
      if (fsStatus.getPermission().toShort() != fsp.toShort()) {
        LOG.info("Explicitly setting permissions to : " + fsp.toShort()
            + ", " + fsp);
        doneDirFc.setPermission(path, fsp);
      }
      existingDoneSubdirs.add(path);
    } catch (FileAlreadyExistsException faeE) { // Nothing to do.
    }
  }
}
 

/**
 * Scans the intermediate directory to find user directories. Scans these for
 * history files if the modification time for the directory has changed. Once
 * it finds history files it starts the process of moving them to the done 
 * directory.
 * 
 * @throws IOException
 *           if there was a error while scanning
 */
void scanIntermediateDirectory() throws IOException {
  // TODO it would be great to limit how often this happens, except in the
  // case where we are looking for a particular job.
  List<FileStatus> userDirList = JobHistoryUtils.localGlobber(
      intermediateDoneDirFc, intermediateDoneDirPath, "");
  LOG.debug("Scanning intermediate dirs");
  for (FileStatus userDir : userDirList) {
    String name = userDir.getPath().getName();
    UserLogDir dir = userDirModificationTimeMap.get(name);
    if(dir == null) {
      dir = new UserLogDir();
      UserLogDir old = userDirModificationTimeMap.putIfAbsent(name, dir);
      if(old != null) {
        dir = old;
      }
    }
    dir.scanIfNeeded(userDir);
  }
}
 

private void addDirectoryToJobListCache(Path path) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Adding " + path + " to job list cache.");
  }
  List<FileStatus> historyFileList = scanDirectoryForHistoryFiles(path,
      doneDirFc);
  for (FileStatus fs : historyFileList) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Adding in history for " + fs.getPath());
    }
    JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
        .getName());
    String confFileName = JobHistoryUtils
        .getIntermediateConfFileName(jobIndexInfo.getJobId());
    String summaryFileName = JobHistoryUtils
        .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
    HistoryFileInfo fileInfo = new HistoryFileInfo(fs.getPath(), new Path(fs
        .getPath().getParent(), confFileName), new Path(fs.getPath()
        .getParent(), summaryFileName), jobIndexInfo, true);
    jobListCache.addIfAbsent(fileInfo);
  }
}
 

private void addDirectoryToSerialNumberIndex(Path serialDirPath) {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Adding " + serialDirPath + " to serial index");
  }
  String serialPart = serialDirPath.getName();
  String timestampPart = JobHistoryUtils
      .getTimestampPartFromPath(serialDirPath.toString());
  if (timestampPart == null) {
    LOG.warn("Could not find timestamp portion from path: " + serialDirPath
        + ". Continuing with next");
    return;
  }
  if (serialPart == null) {
    LOG.warn("Could not find serial portion from path: "
        + serialDirPath.toString() + ". Continuing with next");
  } else {
    serialNumberIndex.add(serialPart, timestampPart);
  }
}
 

private void removeDirectoryFromSerialNumberIndex(Path serialDirPath) {
  String serialPart = serialDirPath.getName();
  String timeStampPart = JobHistoryUtils
      .getTimestampPartFromPath(serialDirPath.toString());
  if (timeStampPart == null) {
    LOG.warn("Could not find timestamp portion from path: "
        + serialDirPath.toString() + ". Continuing with next");
    return;
  }
  if (serialPart == null) {
    LOG.warn("Could not find serial portion from path: "
        + serialDirPath.toString() + ". Continuing with next");
    return;
  }
  serialNumberIndex.remove(serialPart, timeStampPart);
}
 

/**
 * Searches the job history file FileStatus list for the specified JobId.
 * 
 * @param fileStatusList
 *          fileStatus list of Job History Files.
 * @param jobId
 *          The JobId to find.
 * @return A FileInfo object for the jobId, null if not found.
 * @throws IOException
 */
private HistoryFileInfo getJobFileInfo(List<FileStatus> fileStatusList,
    JobId jobId) throws IOException {
  for (FileStatus fs : fileStatusList) {
    JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
        .getName());
    if (jobIndexInfo.getJobId().equals(jobId)) {
      String confFileName = JobHistoryUtils
          .getIntermediateConfFileName(jobIndexInfo.getJobId());
      String summaryFileName = JobHistoryUtils
          .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
      HistoryFileInfo fileInfo = new HistoryFileInfo(fs.getPath(), new Path(
          fs.getPath().getParent(), confFileName), new Path(fs.getPath()
          .getParent(), summaryFileName), jobIndexInfo, true);
      return fileInfo;
    }
  }
  return null;
}
 

/**
 * Scans old directories known by the idToDateString map for the specified
 * jobId. If the number of directories is higher than the supported size of
 * the idToDateString cache, the jobId will not be found.
 * 
 * @param jobId
 *          the jobId.
 * @return
 * @throws IOException
 */
private HistoryFileInfo scanOldDirsForJob(JobId jobId) throws IOException {
  String boxedSerialNumber = JobHistoryUtils.serialNumberDirectoryComponent(
      jobId, serialNumberFormat);
  Set<String> dateStringSet = serialNumberIndex.get(boxedSerialNumber);
  if (dateStringSet == null) {
    return null;
  }
  for (String timestampPart : dateStringSet) {
    Path logDir = canonicalHistoryLogPath(jobId, timestampPart);
    List<FileStatus> fileStatusList = scanDirectoryForHistoryFiles(logDir,
        doneDirFc);
    HistoryFileInfo fileInfo = getJobFileInfo(fileStatusList, jobId);
    if (fileInfo != null) {
      return fileInfo;
    }
  }
  return null;
}
 

@Test
public void testGetHistoryIntermediateDoneDirForUser() throws IOException {
  // Test relative path
  Configuration conf = new Configuration();
  conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR,
      "/mapred/history/done_intermediate");
  conf.set(MRJobConfig.USER_NAME, System.getProperty("user.name"));
  String pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
  Assert.assertEquals("/mapred/history/done_intermediate/" +
      System.getProperty("user.name"), pathStr);

  // Test fully qualified path
  // Create default configuration pointing to the minicluster
  conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
      dfsCluster.getURI().toString());
  FileOutputStream os = new FileOutputStream(coreSitePath);
  conf.writeXml(os);
  os.close();
  // Simulate execution under a non-default namenode
  conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
          "file:///");
  pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
  Assert.assertEquals(dfsCluster.getURI().toString() +
      "/mapred/history/done_intermediate/" + System.getProperty("user.name"),
      pathStr);
}
 

public static FSDataInputStream getPreviousJobHistoryFileStream(
    Configuration conf, ApplicationAttemptId applicationAttemptId)
    throws IOException {
  FSDataInputStream in = null;
  Path historyFile = null;
  String jobId =
      TypeConverter.fromYarn(applicationAttemptId.getApplicationId())
        .toString();
  String jobhistoryDir =
      JobHistoryUtils.getConfiguredHistoryStagingDirPrefix(conf, jobId);
  Path histDirPath =
      FileContext.getFileContext(conf).makeQualified(new Path(jobhistoryDir));
  FileContext fc = FileContext.getFileContext(histDirPath.toUri(), conf);
  // read the previous history file
  historyFile =
      fc.makeQualified(JobHistoryUtils.getStagingJobHistoryFile(histDirPath,
        jobId, (applicationAttemptId.getAttemptId() - 1)));
  LOG.info("History file is at " + historyFile);
  in = fc.open(historyFile);
  return in;
}
 

private void moveToDoneNow(Path fromPath, Path toPath) throws IOException {
  // check if path exists, in case of retries it may not exist
  if (stagingDirFS.exists(fromPath)) {
    LOG.info("Copying " + fromPath.toString() + " to " + toPath.toString());
    // TODO temporarily removing the existing dst
    if (doneDirFS.exists(toPath)) {
      doneDirFS.delete(toPath, true);
    }
    boolean copied = FileUtil.copy(stagingDirFS, fromPath, doneDirFS, toPath,
        false, getConfig());

    if (copied)
      LOG.info("Copied to done location: " + toPath);
    else 
      LOG.info("copy failed");
    doneDirFS.setPermission(toPath, new FsPermission(
        JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS));
  }
}
 

private void makeDoneSubdir(Path path) throws IOException {
  try {
    doneDirFc.getFileStatus(path);
    existingDoneSubdirs.add(path);
  } catch (FileNotFoundException fnfE) {
    try {
      FsPermission fsp = new FsPermission(
          JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION);
      doneDirFc.mkdir(path, fsp, true);
      FileStatus fsStatus = doneDirFc.getFileStatus(path);
      LOG.info("Perms after creating " + fsStatus.getPermission().toShort()
          + ", Expected: " + fsp.toShort());
      if (fsStatus.getPermission().toShort() != fsp.toShort()) {
        LOG.info("Explicitly setting permissions to : " + fsp.toShort()
            + ", " + fsp);
        doneDirFc.setPermission(path, fsp);
      }
      existingDoneSubdirs.add(path);
    } catch (FileAlreadyExistsException faeE) { // Nothing to do.
    }
  }
}
 

/**
 * Scans old directories known by the idToDateString map for the specified
 * jobId. If the number of directories is higher than the supported size of
 * the idToDateString cache, the jobId will not be found.
 * 
 * @param jobId
 *          the jobId.
 * @return
 * @throws IOException
 */
private HistoryFileInfo scanOldDirsForJob(JobId jobId) throws IOException {
  String boxedSerialNumber = JobHistoryUtils.serialNumberDirectoryComponent(
      jobId, serialNumberFormat);
  Set<String> dateStringSet = serialNumberIndex.get(boxedSerialNumber);
  if (dateStringSet == null) {
    return null;
  }
  for (String timestampPart : dateStringSet) {
    Path logDir = canonicalHistoryLogPath(jobId, timestampPart);
    List<FileStatus> fileStatusList = scanDirectoryForHistoryFiles(logDir,
        doneDirFc);
    HistoryFileInfo fileInfo = getJobFileInfo(fileStatusList, jobId);
    if (fileInfo != null) {
      return fileInfo;
    }
  }
  return null;
}
 

/**
 * Searches the job history file FileStatus list for the specified JobId.
 * 
 * @param fileStatusList
 *          fileStatus list of Job History Files.
 * @param jobId
 *          The JobId to find.
 * @return A FileInfo object for the jobId, null if not found.
 * @throws IOException
 */
private HistoryFileInfo getJobFileInfo(List<FileStatus> fileStatusList,
    JobId jobId) throws IOException {
  for (FileStatus fs : fileStatusList) {
    JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
        .getName());
    if (jobIndexInfo.getJobId().equals(jobId)) {
      String confFileName = JobHistoryUtils
          .getIntermediateConfFileName(jobIndexInfo.getJobId());
      String summaryFileName = JobHistoryUtils
          .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
      HistoryFileInfo fileInfo = createHistoryFileInfo(fs.getPath(), new Path(
          fs.getPath().getParent(), confFileName), new Path(fs.getPath()
          .getParent(), summaryFileName), jobIndexInfo, true);
      return fileInfo;
    }
  }
  return null;
}
 

/**
 * Scans the intermediate directory to find user directories. Scans these for
 * history files if the modification time for the directory has changed. Once
 * it finds history files it starts the process of moving them to the done 
 * directory.
 * 
 * @throws IOException
 *           if there was a error while scanning
 */
void scanIntermediateDirectory() throws IOException {
  // TODO it would be great to limit how often this happens, except in the
  // case where we are looking for a particular job.
  List<FileStatus> userDirList = JobHistoryUtils.localGlobber(
      intermediateDoneDirFc, intermediateDoneDirPath, "");
  LOG.debug("Scanning intermediate dirs");
  for (FileStatus userDir : userDirList) {
    String name = userDir.getPath().getName();
    UserLogDir dir = userDirModificationTimeMap.get(name);
    if(dir == null) {
      dir = new UserLogDir();
      UserLogDir old = userDirModificationTimeMap.putIfAbsent(name, dir);
      if(old != null) {
        dir = old;
      }
    }
    dir.scanIfNeeded(userDir);
  }
}
 

private void moveToDoneNow(Path fromPath, Path toPath) throws IOException {
  // check if path exists, in case of retries it may not exist
  if (stagingDirFS.exists(fromPath)) {
    LOG.info("Copying " + fromPath.toString() + " to " + toPath.toString());
    // TODO temporarily removing the existing dst
    if (doneDirFS.exists(toPath)) {
      doneDirFS.delete(toPath, true);
    }
    boolean copied = FileUtil.copy(stagingDirFS, fromPath, doneDirFS, toPath,
        false, getConfig());

    if (copied)
      LOG.info("Copied to done location: " + toPath);
    else 
      LOG.info("copy failed");
    doneDirFS.setPermission(toPath, new FsPermission(
        JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS));
  }
}
 

public static FSDataInputStream getPreviousJobHistoryFileStream(
    Configuration conf, ApplicationAttemptId applicationAttemptId)
    throws IOException {
  FSDataInputStream in = null;
  Path historyFile = null;
  String jobId =
      TypeConverter.fromYarn(applicationAttemptId.getApplicationId())
        .toString();
  String jobhistoryDir =
      JobHistoryUtils.getConfiguredHistoryStagingDirPrefix(conf, jobId);
  Path histDirPath =
      FileContext.getFileContext(conf).makeQualified(new Path(jobhistoryDir));
  FileContext fc = FileContext.getFileContext(histDirPath.toUri(), conf);
  // read the previous history file
  historyFile =
      fc.makeQualified(JobHistoryUtils.getStagingJobHistoryFile(histDirPath,
        jobId, (applicationAttemptId.getAttemptId() - 1)));
  LOG.info("History file is at " + historyFile);
  in = fc.open(historyFile);
  return in;
}
 

@Test
public void testGetHistoryIntermediateDoneDirForUser() throws IOException {
  // Test relative path
  Configuration conf = new Configuration();
  conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR,
      "/mapred/history/done_intermediate");
  conf.set(MRJobConfig.USER_NAME, System.getProperty("user.name"));
  String pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
  Assert.assertEquals("/mapred/history/done_intermediate/" +
      System.getProperty("user.name"), pathStr);

  // Test fully qualified path
  // Create default configuration pointing to the minicluster
  conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
      dfsCluster.getURI().toString());
  FileOutputStream os = new FileOutputStream(coreSitePath);
  conf.writeXml(os);
  os.close();
  // Simulate execution under a non-default namenode
  conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
          "file:///");
  pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
  Assert.assertEquals(dfsCluster.getURI().toString() +
      "/mapred/history/done_intermediate/" + System.getProperty("user.name"),
      pathStr);
}
 

private void removeDirectoryFromSerialNumberIndex(Path serialDirPath) {
  String serialPart = serialDirPath.getName();
  String timeStampPart = JobHistoryUtils
      .getTimestampPartFromPath(serialDirPath.toString());
  if (timeStampPart == null) {
    LOG.warn("Could not find timestamp portion from path: "
        + serialDirPath.toString() + ". Continuing with next");
    return;
  }
  if (serialPart == null) {
    LOG.warn("Could not find serial portion from path: "
        + serialDirPath.toString() + ". Continuing with next");
    return;
  }
  serialNumberIndex.remove(serialPart, timeStampPart);
}
 

private void addDirectoryToSerialNumberIndex(Path serialDirPath) {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Adding " + serialDirPath + " to serial index");
  }
  String serialPart = serialDirPath.getName();
  String timestampPart = JobHistoryUtils
      .getTimestampPartFromPath(serialDirPath.toString());
  if (timestampPart == null) {
    LOG.warn("Could not find timestamp portion from path: " + serialDirPath
        + ". Continuing with next");
    return;
  }
  if (serialPart == null) {
    LOG.warn("Could not find serial portion from path: "
        + serialDirPath.toString() + ". Continuing with next");
  } else {
    serialNumberIndex.add(serialPart, timestampPart);
  }
}
 

private void addDirectoryToJobListCache(Path path) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Adding " + path + " to job list cache.");
  }
  List<FileStatus> historyFileList = scanDirectoryForHistoryFiles(path,
      doneDirFc);
  for (FileStatus fs : historyFileList) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Adding in history for " + fs.getPath());
    }
    JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
        .getName());
    String confFileName = JobHistoryUtils
        .getIntermediateConfFileName(jobIndexInfo.getJobId());
    String summaryFileName = JobHistoryUtils
        .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
    HistoryFileInfo fileInfo = createHistoryFileInfo(fs.getPath(), new Path(fs
        .getPath().getParent(), confFileName), new Path(fs.getPath()
        .getParent(), summaryFileName), jobIndexInfo, true);
    jobListCache.addIfAbsent(fileInfo);
  }
}
 

private static FSDataInputStream getPreviousJobHistoryStream(
    Configuration conf, ApplicationAttemptId appAttemptId)
    throws IOException {
  Path historyFile = JobHistoryUtils.getPreviousJobHistoryPath(conf,
      appAttemptId);
  LOG.info("Previous history file is at " + historyFile);
  return historyFile.getFileSystem(conf).open(historyFile);
}
 

@Override
protected void serviceInit(Configuration conf) throws Exception {
  this.conf = conf;

  int serialNumberLowDigits = 3;
  serialNumberFormat = ("%0"
      + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits)
      + "d");

  long maxFSWaitTime = conf.getLong(
      JHAdminConfig.MR_HISTORY_MAX_START_WAIT_TIME,
      JHAdminConfig.DEFAULT_MR_HISTORY_MAX_START_WAIT_TIME);
  createHistoryDirs(new SystemClock(), 10 * 1000, maxFSWaitTime);

  this.aclsMgr = new JobACLsManager(conf);

  maxHistoryAge = conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS,
      JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);
  
  jobListCache = createJobListCache();

  serialNumberIndex = new SerialNumberIndex(conf.getInt(
      JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE,
      JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE));

  int numMoveThreads = conf.getInt(
      JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT,
      JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT);
  ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat(
      "MoveIntermediateToDone Thread #%d").build();
  moveToDoneExecutor = new ThreadPoolExecutor(numMoveThreads, numMoveThreads,
      1, TimeUnit.HOURS, new LinkedBlockingQueue<Runnable>(), tf);

  super.serviceInit(conf);
}
 

@Override
protected void serviceInit(Configuration conf) throws Exception {
  this.conf = conf;

  int serialNumberLowDigits = 3;
  serialNumberFormat = ("%0"
      + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits)
      + "d");

  long maxFSWaitTime = conf.getLong(
      JHAdminConfig.MR_HISTORY_MAX_START_WAIT_TIME,
      JHAdminConfig.DEFAULT_MR_HISTORY_MAX_START_WAIT_TIME);
  createHistoryDirs(new SystemClock(), 10 * 1000, maxFSWaitTime);

  this.aclsMgr = new JobACLsManager(conf);

  maxHistoryAge = conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS,
      JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);
  
  jobListCache = createJobListCache();

  serialNumberIndex = new SerialNumberIndex(conf.getInt(
      JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE,
      JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE));

  int numMoveThreads = conf.getInt(
      JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT,
      JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT);
  moveToDoneExecutor = createMoveToDoneThreadPool(numMoveThreads);
  super.serviceInit(conf);
}
 

private static FSDataInputStream getPreviousJobHistoryStream(
    Configuration conf, ApplicationAttemptId appAttemptId)
    throws IOException {
  Path historyFile = JobHistoryUtils.getPreviousJobHistoryPath(conf,
      appAttemptId);
  LOG.info("Previous history file is at " + historyFile);
  return historyFile.getFileSystem(conf).open(historyFile);
}
 

private Path canonicalHistoryLogPath(JobId id, String timestampComponent) {
  return new Path(doneDirPrefixPath, JobHistoryUtils.historyLogSubdirectory(
      id, timestampComponent, serialNumberFormat));
}
 

private Path canonicalHistoryLogPath(JobId id, long millisecondTime) {
  String timestampComponent = JobHistoryUtils
      .timestampDirectoryComponent(millisecondTime);
  return new Path(doneDirPrefixPath, JobHistoryUtils.historyLogSubdirectory(
      id, timestampComponent, serialNumberFormat));
}
 

@Override
public void serviceInit(Configuration conf) throws Exception {
  conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_TEZ_FRAMEWORK_NAME);
  // blacklisting disabled to prevent scheduling issues
  conf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
  if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) {
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(getTestWorkDir(),
        "apps_staging_dir" + Path.SEPARATOR).getAbsolutePath());
  }
  
  if (conf.get(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC) == null) {
    // nothing defined. set quick delete value
    conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 0l);
  }
  
  File appJarLocalFile = new File(MiniTezCluster.APPJAR);

  if (!appJarLocalFile.exists()) {
    String message = "TezAppJar " + MiniTezCluster.APPJAR
        + " not found. Exiting.";
    LOG.info(message);
    throw new TezUncheckedException(message);
  }
  
  FileSystem fs = FileSystem.get(conf);
  Path testRootDir = fs.makeQualified(new Path("target", getName() + "-tmpDir"));
  Path appRemoteJar = new Path(testRootDir, "TezAppJar.jar");
  // Copy AppJar and make it public.
  Path appMasterJar = new Path(MiniTezCluster.APPJAR);
  fs.copyFromLocalFile(appMasterJar, appRemoteJar);
  fs.setPermission(appRemoteJar, new FsPermission("777"));

  conf.set(TezConfiguration.TEZ_LIB_URIS, appRemoteJar.toUri().toString());
  LOG.info("Set TEZ-LIB-URI to: " + conf.get(TezConfiguration.TEZ_LIB_URIS));

  // VMEM monitoring disabled, PMEM monitoring enabled.
  conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
  conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);

  conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,  "000");

  try {
    Path stagingPath = FileContext.getFileContext(conf).makeQualified(
        new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR)));
    /*
     * Re-configure the staging path on Windows if the file system is localFs.
     * We need to use a absolute path that contains the drive letter. The unit
     * test could run on a different drive than the AM. We can run into the
     * issue that job files are localized to the drive where the test runs on,
     * while the AM starts on a different drive and fails to find the job
     * metafiles. Using absolute path can avoid this ambiguity.
     */
    if (Path.WINDOWS) {
      if (LocalFileSystem.class.isInstance(stagingPath.getFileSystem(conf))) {
        conf.set(MRJobConfig.MR_AM_STAGING_DIR,
            new File(conf.get(MRJobConfig.MR_AM_STAGING_DIR))
                .getAbsolutePath());
      }
    }
    FileContext fc=FileContext.getFileContext(stagingPath.toUri(), conf);
    if (fc.util().exists(stagingPath)) {
      LOG.info(stagingPath + " exists! deleting...");
      fc.delete(stagingPath, true);
    }
    LOG.info("mkdir: " + stagingPath);
    fc.mkdir(stagingPath, null, true);

    //mkdir done directory as well
    String doneDir =
        JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
    Path doneDirPath = fc.makeQualified(new Path(doneDir));
    fc.mkdir(doneDirPath, null, true);
  } catch (IOException e) {
    throw new TezUncheckedException("Could not create staging directory. ", e);
  }
  conf.set(MRConfig.MASTER_ADDRESS, "test");

  //configure the shuffle service in NM
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
      new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT,
      ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class,
      Service.class);

  // Non-standard shuffle port
  conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);

  conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR,
      DefaultContainerExecutor.class, ContainerExecutor.class);

  // TestMRJobs is for testing non-uberized operation only; see TestUberAM
  // for corresponding uberized tests.
  conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
  super.serviceInit(conf);
}
 

@Override
public String getJobHistoryDir() throws IOException, InterruptedException {
  return JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
}
 

List<FileStatus> getHistoryDirsForCleaning(long cutoff) throws IOException {
    return JobHistoryUtils.
      getHistoryDirsForCleaning(doneDirFc, doneDirPrefixPath, cutoff);
}
 

/**
 * Clean up older history files.
 * 
 * @throws IOException
 *           on any error trying to remove the entries.
 */
@SuppressWarnings("unchecked")
void clean() throws IOException {
  long cutoff = System.currentTimeMillis() - maxHistoryAge;
  boolean halted = false;
  List<FileStatus> serialDirList = getHistoryDirsForCleaning(cutoff);
  // Sort in ascending order. Relies on YYYY/MM/DD/Serial
  Collections.sort(serialDirList);
  for (FileStatus serialDir : serialDirList) {
    List<FileStatus> historyFileList = scanDirectoryForHistoryFiles(
        serialDir.getPath(), doneDirFc);
    for (FileStatus historyFile : historyFileList) {
      JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(historyFile
          .getPath().getName());
      long effectiveTimestamp = getEffectiveTimestamp(
          jobIndexInfo.getFinishTime(), historyFile);
      if (effectiveTimestamp <= cutoff) {
        HistoryFileInfo fileInfo = this.jobListCache.get(jobIndexInfo
            .getJobId());
        if (fileInfo == null) {
          String confFileName = JobHistoryUtils
              .getIntermediateConfFileName(jobIndexInfo.getJobId());

          fileInfo = new HistoryFileInfo(historyFile.getPath(), new Path(
              historyFile.getPath().getParent(), confFileName), null,
              jobIndexInfo, true);
        }
        deleteJobFromDone(fileInfo);
      } else {
        halted = true;
        break;
      }
    }
    if (!halted) {
      deleteDir(serialDir);
      removeDirectoryFromSerialNumberIndex(serialDir.getPath());
      existingDoneSubdirs.remove(serialDir.getPath());
    } else {
      break; // Don't scan any more directories.
    }
  }
}