org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils Java Examples

The following examples show how to use org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HistoryFileManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void makeDoneSubdir(Path path) throws IOException {
  try {
    doneDirFc.getFileStatus(path);
    existingDoneSubdirs.add(path);
  } catch (FileNotFoundException fnfE) {
    try {
      FsPermission fsp = new FsPermission(
          JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION);
      doneDirFc.mkdir(path, fsp, true);
      FileStatus fsStatus = doneDirFc.getFileStatus(path);
      LOG.info("Perms after creating " + fsStatus.getPermission().toShort()
          + ", Expected: " + fsp.toShort());
      if (fsStatus.getPermission().toShort() != fsp.toShort()) {
        LOG.info("Explicitly setting permissions to : " + fsp.toShort()
            + ", " + fsp);
        doneDirFc.setPermission(path, fsp);
      }
      existingDoneSubdirs.add(path);
    } catch (FileAlreadyExistsException faeE) { // Nothing to do.
    }
  }
}
 
Example #2
Source File: HistoryFileManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Scans the intermediate directory to find user directories. Scans these for
 * history files if the modification time for the directory has changed. Once
 * it finds history files it starts the process of moving them to the done 
 * directory.
 * 
 * @throws IOException
 *           if there was a error while scanning
 */
void scanIntermediateDirectory() throws IOException {
  // TODO it would be great to limit how often this happens, except in the
  // case where we are looking for a particular job.
  List<FileStatus> userDirList = JobHistoryUtils.localGlobber(
      intermediateDoneDirFc, intermediateDoneDirPath, "");
  LOG.debug("Scanning intermediate dirs");
  for (FileStatus userDir : userDirList) {
    String name = userDir.getPath().getName();
    UserLogDir dir = userDirModificationTimeMap.get(name);
    if(dir == null) {
      dir = new UserLogDir();
      UserLogDir old = userDirModificationTimeMap.putIfAbsent(name, dir);
      if(old != null) {
        dir = old;
      }
    }
    dir.scanIfNeeded(userDir);
  }
}
 
Example #3
Source File: HistoryFileManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void addDirectoryToJobListCache(Path path) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Adding " + path + " to job list cache.");
  }
  List<FileStatus> historyFileList = scanDirectoryForHistoryFiles(path,
      doneDirFc);
  for (FileStatus fs : historyFileList) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Adding in history for " + fs.getPath());
    }
    JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
        .getName());
    String confFileName = JobHistoryUtils
        .getIntermediateConfFileName(jobIndexInfo.getJobId());
    String summaryFileName = JobHistoryUtils
        .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
    HistoryFileInfo fileInfo = new HistoryFileInfo(fs.getPath(), new Path(fs
        .getPath().getParent(), confFileName), new Path(fs.getPath()
        .getParent(), summaryFileName), jobIndexInfo, true);
    jobListCache.addIfAbsent(fileInfo);
  }
}
 
Example #4
Source File: HistoryFileManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void addDirectoryToSerialNumberIndex(Path serialDirPath) {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Adding " + serialDirPath + " to serial index");
  }
  String serialPart = serialDirPath.getName();
  String timestampPart = JobHistoryUtils
      .getTimestampPartFromPath(serialDirPath.toString());
  if (timestampPart == null) {
    LOG.warn("Could not find timestamp portion from path: " + serialDirPath
        + ". Continuing with next");
    return;
  }
  if (serialPart == null) {
    LOG.warn("Could not find serial portion from path: "
        + serialDirPath.toString() + ". Continuing with next");
  } else {
    serialNumberIndex.add(serialPart, timestampPart);
  }
}
 
Example #5
Source File: HistoryFileManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void removeDirectoryFromSerialNumberIndex(Path serialDirPath) {
  String serialPart = serialDirPath.getName();
  String timeStampPart = JobHistoryUtils
      .getTimestampPartFromPath(serialDirPath.toString());
  if (timeStampPart == null) {
    LOG.warn("Could not find timestamp portion from path: "
        + serialDirPath.toString() + ". Continuing with next");
    return;
  }
  if (serialPart == null) {
    LOG.warn("Could not find serial portion from path: "
        + serialDirPath.toString() + ". Continuing with next");
    return;
  }
  serialNumberIndex.remove(serialPart, timeStampPart);
}
 
Example #6
Source File: HistoryFileManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Searches the job history file FileStatus list for the specified JobId.
 * 
 * @param fileStatusList
 *          fileStatus list of Job History Files.
 * @param jobId
 *          The JobId to find.
 * @return A FileInfo object for the jobId, null if not found.
 * @throws IOException
 */
private HistoryFileInfo getJobFileInfo(List<FileStatus> fileStatusList,
    JobId jobId) throws IOException {
  for (FileStatus fs : fileStatusList) {
    JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
        .getName());
    if (jobIndexInfo.getJobId().equals(jobId)) {
      String confFileName = JobHistoryUtils
          .getIntermediateConfFileName(jobIndexInfo.getJobId());
      String summaryFileName = JobHistoryUtils
          .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
      HistoryFileInfo fileInfo = new HistoryFileInfo(fs.getPath(), new Path(
          fs.getPath().getParent(), confFileName), new Path(fs.getPath()
          .getParent(), summaryFileName), jobIndexInfo, true);
      return fileInfo;
    }
  }
  return null;
}
 
Example #7
Source File: HistoryFileManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Scans old directories known by the idToDateString map for the specified
 * jobId. If the number of directories is higher than the supported size of
 * the idToDateString cache, the jobId will not be found.
 * 
 * @param jobId
 *          the jobId.
 * @return
 * @throws IOException
 */
private HistoryFileInfo scanOldDirsForJob(JobId jobId) throws IOException {
  String boxedSerialNumber = JobHistoryUtils.serialNumberDirectoryComponent(
      jobId, serialNumberFormat);
  Set<String> dateStringSet = serialNumberIndex.get(boxedSerialNumber);
  if (dateStringSet == null) {
    return null;
  }
  for (String timestampPart : dateStringSet) {
    Path logDir = canonicalHistoryLogPath(jobId, timestampPart);
    List<FileStatus> fileStatusList = scanDirectoryForHistoryFiles(logDir,
        doneDirFc);
    HistoryFileInfo fileInfo = getJobFileInfo(fileStatusList, jobId);
    if (fileInfo != null) {
      return fileInfo;
    }
  }
  return null;
}
 
Example #8
Source File: TestJobHistoryEventHandler.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetHistoryIntermediateDoneDirForUser() throws IOException {
  // Test relative path
  Configuration conf = new Configuration();
  conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR,
      "/mapred/history/done_intermediate");
  conf.set(MRJobConfig.USER_NAME, System.getProperty("user.name"));
  String pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
  Assert.assertEquals("/mapred/history/done_intermediate/" +
      System.getProperty("user.name"), pathStr);

  // Test fully qualified path
  // Create default configuration pointing to the minicluster
  conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
      dfsCluster.getURI().toString());
  FileOutputStream os = new FileOutputStream(coreSitePath);
  conf.writeXml(os);
  os.close();
  // Simulate execution under a non-default namenode
  conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
          "file:///");
  pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
  Assert.assertEquals(dfsCluster.getURI().toString() +
      "/mapred/history/done_intermediate/" + System.getProperty("user.name"),
      pathStr);
}
 
Example #9
Source File: JobHistoryCopyService.java    From big-c with Apache License 2.0 6 votes vote down vote up
public static FSDataInputStream getPreviousJobHistoryFileStream(
    Configuration conf, ApplicationAttemptId applicationAttemptId)
    throws IOException {
  FSDataInputStream in = null;
  Path historyFile = null;
  String jobId =
      TypeConverter.fromYarn(applicationAttemptId.getApplicationId())
        .toString();
  String jobhistoryDir =
      JobHistoryUtils.getConfiguredHistoryStagingDirPrefix(conf, jobId);
  Path histDirPath =
      FileContext.getFileContext(conf).makeQualified(new Path(jobhistoryDir));
  FileContext fc = FileContext.getFileContext(histDirPath.toUri(), conf);
  // read the previous history file
  historyFile =
      fc.makeQualified(JobHistoryUtils.getStagingJobHistoryFile(histDirPath,
        jobId, (applicationAttemptId.getAttemptId() - 1)));
  LOG.info("History file is at " + historyFile);
  in = fc.open(historyFile);
  return in;
}
 
Example #10
Source File: JobHistoryEventHandler.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void moveToDoneNow(Path fromPath, Path toPath) throws IOException {
  // check if path exists, in case of retries it may not exist
  if (stagingDirFS.exists(fromPath)) {
    LOG.info("Copying " + fromPath.toString() + " to " + toPath.toString());
    // TODO temporarily removing the existing dst
    if (doneDirFS.exists(toPath)) {
      doneDirFS.delete(toPath, true);
    }
    boolean copied = FileUtil.copy(stagingDirFS, fromPath, doneDirFS, toPath,
        false, getConfig());

    if (copied)
      LOG.info("Copied to done location: " + toPath);
    else 
      LOG.info("copy failed");
    doneDirFS.setPermission(toPath, new FsPermission(
        JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS));
  }
}
 
Example #11
Source File: HistoryFileManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void makeDoneSubdir(Path path) throws IOException {
  try {
    doneDirFc.getFileStatus(path);
    existingDoneSubdirs.add(path);
  } catch (FileNotFoundException fnfE) {
    try {
      FsPermission fsp = new FsPermission(
          JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION);
      doneDirFc.mkdir(path, fsp, true);
      FileStatus fsStatus = doneDirFc.getFileStatus(path);
      LOG.info("Perms after creating " + fsStatus.getPermission().toShort()
          + ", Expected: " + fsp.toShort());
      if (fsStatus.getPermission().toShort() != fsp.toShort()) {
        LOG.info("Explicitly setting permissions to : " + fsp.toShort()
            + ", " + fsp);
        doneDirFc.setPermission(path, fsp);
      }
      existingDoneSubdirs.add(path);
    } catch (FileAlreadyExistsException faeE) { // Nothing to do.
    }
  }
}
 
Example #12
Source File: HistoryFileManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Scans old directories known by the idToDateString map for the specified
 * jobId. If the number of directories is higher than the supported size of
 * the idToDateString cache, the jobId will not be found.
 * 
 * @param jobId
 *          the jobId.
 * @return
 * @throws IOException
 */
private HistoryFileInfo scanOldDirsForJob(JobId jobId) throws IOException {
  String boxedSerialNumber = JobHistoryUtils.serialNumberDirectoryComponent(
      jobId, serialNumberFormat);
  Set<String> dateStringSet = serialNumberIndex.get(boxedSerialNumber);
  if (dateStringSet == null) {
    return null;
  }
  for (String timestampPart : dateStringSet) {
    Path logDir = canonicalHistoryLogPath(jobId, timestampPart);
    List<FileStatus> fileStatusList = scanDirectoryForHistoryFiles(logDir,
        doneDirFc);
    HistoryFileInfo fileInfo = getJobFileInfo(fileStatusList, jobId);
    if (fileInfo != null) {
      return fileInfo;
    }
  }
  return null;
}
 
Example #13
Source File: HistoryFileManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Searches the job history file FileStatus list for the specified JobId.
 * 
 * @param fileStatusList
 *          fileStatus list of Job History Files.
 * @param jobId
 *          The JobId to find.
 * @return A FileInfo object for the jobId, null if not found.
 * @throws IOException
 */
private HistoryFileInfo getJobFileInfo(List<FileStatus> fileStatusList,
    JobId jobId) throws IOException {
  for (FileStatus fs : fileStatusList) {
    JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
        .getName());
    if (jobIndexInfo.getJobId().equals(jobId)) {
      String confFileName = JobHistoryUtils
          .getIntermediateConfFileName(jobIndexInfo.getJobId());
      String summaryFileName = JobHistoryUtils
          .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
      HistoryFileInfo fileInfo = createHistoryFileInfo(fs.getPath(), new Path(
          fs.getPath().getParent(), confFileName), new Path(fs.getPath()
          .getParent(), summaryFileName), jobIndexInfo, true);
      return fileInfo;
    }
  }
  return null;
}
 
Example #14
Source File: HistoryFileManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Scans the intermediate directory to find user directories. Scans these for
 * history files if the modification time for the directory has changed. Once
 * it finds history files it starts the process of moving them to the done 
 * directory.
 * 
 * @throws IOException
 *           if there was a error while scanning
 */
void scanIntermediateDirectory() throws IOException {
  // TODO it would be great to limit how often this happens, except in the
  // case where we are looking for a particular job.
  List<FileStatus> userDirList = JobHistoryUtils.localGlobber(
      intermediateDoneDirFc, intermediateDoneDirPath, "");
  LOG.debug("Scanning intermediate dirs");
  for (FileStatus userDir : userDirList) {
    String name = userDir.getPath().getName();
    UserLogDir dir = userDirModificationTimeMap.get(name);
    if(dir == null) {
      dir = new UserLogDir();
      UserLogDir old = userDirModificationTimeMap.putIfAbsent(name, dir);
      if(old != null) {
        dir = old;
      }
    }
    dir.scanIfNeeded(userDir);
  }
}
 
Example #15
Source File: JobHistoryEventHandler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void moveToDoneNow(Path fromPath, Path toPath) throws IOException {
  // check if path exists, in case of retries it may not exist
  if (stagingDirFS.exists(fromPath)) {
    LOG.info("Copying " + fromPath.toString() + " to " + toPath.toString());
    // TODO temporarily removing the existing dst
    if (doneDirFS.exists(toPath)) {
      doneDirFS.delete(toPath, true);
    }
    boolean copied = FileUtil.copy(stagingDirFS, fromPath, doneDirFS, toPath,
        false, getConfig());

    if (copied)
      LOG.info("Copied to done location: " + toPath);
    else 
      LOG.info("copy failed");
    doneDirFS.setPermission(toPath, new FsPermission(
        JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS));
  }
}
 
Example #16
Source File: JobHistoryCopyService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public static FSDataInputStream getPreviousJobHistoryFileStream(
    Configuration conf, ApplicationAttemptId applicationAttemptId)
    throws IOException {
  FSDataInputStream in = null;
  Path historyFile = null;
  String jobId =
      TypeConverter.fromYarn(applicationAttemptId.getApplicationId())
        .toString();
  String jobhistoryDir =
      JobHistoryUtils.getConfiguredHistoryStagingDirPrefix(conf, jobId);
  Path histDirPath =
      FileContext.getFileContext(conf).makeQualified(new Path(jobhistoryDir));
  FileContext fc = FileContext.getFileContext(histDirPath.toUri(), conf);
  // read the previous history file
  historyFile =
      fc.makeQualified(JobHistoryUtils.getStagingJobHistoryFile(histDirPath,
        jobId, (applicationAttemptId.getAttemptId() - 1)));
  LOG.info("History file is at " + historyFile);
  in = fc.open(historyFile);
  return in;
}
 
Example #17
Source File: TestJobHistoryEventHandler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetHistoryIntermediateDoneDirForUser() throws IOException {
  // Test relative path
  Configuration conf = new Configuration();
  conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR,
      "/mapred/history/done_intermediate");
  conf.set(MRJobConfig.USER_NAME, System.getProperty("user.name"));
  String pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
  Assert.assertEquals("/mapred/history/done_intermediate/" +
      System.getProperty("user.name"), pathStr);

  // Test fully qualified path
  // Create default configuration pointing to the minicluster
  conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
      dfsCluster.getURI().toString());
  FileOutputStream os = new FileOutputStream(coreSitePath);
  conf.writeXml(os);
  os.close();
  // Simulate execution under a non-default namenode
  conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
          "file:///");
  pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
  Assert.assertEquals(dfsCluster.getURI().toString() +
      "/mapred/history/done_intermediate/" + System.getProperty("user.name"),
      pathStr);
}
 
Example #18
Source File: HistoryFileManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void removeDirectoryFromSerialNumberIndex(Path serialDirPath) {
  String serialPart = serialDirPath.getName();
  String timeStampPart = JobHistoryUtils
      .getTimestampPartFromPath(serialDirPath.toString());
  if (timeStampPart == null) {
    LOG.warn("Could not find timestamp portion from path: "
        + serialDirPath.toString() + ". Continuing with next");
    return;
  }
  if (serialPart == null) {
    LOG.warn("Could not find serial portion from path: "
        + serialDirPath.toString() + ". Continuing with next");
    return;
  }
  serialNumberIndex.remove(serialPart, timeStampPart);
}
 
Example #19
Source File: HistoryFileManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void addDirectoryToSerialNumberIndex(Path serialDirPath) {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Adding " + serialDirPath + " to serial index");
  }
  String serialPart = serialDirPath.getName();
  String timestampPart = JobHistoryUtils
      .getTimestampPartFromPath(serialDirPath.toString());
  if (timestampPart == null) {
    LOG.warn("Could not find timestamp portion from path: " + serialDirPath
        + ". Continuing with next");
    return;
  }
  if (serialPart == null) {
    LOG.warn("Could not find serial portion from path: "
        + serialDirPath.toString() + ". Continuing with next");
  } else {
    serialNumberIndex.add(serialPart, timestampPart);
  }
}
 
Example #20
Source File: HistoryFileManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void addDirectoryToJobListCache(Path path) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Adding " + path + " to job list cache.");
  }
  List<FileStatus> historyFileList = scanDirectoryForHistoryFiles(path,
      doneDirFc);
  for (FileStatus fs : historyFileList) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Adding in history for " + fs.getPath());
    }
    JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
        .getName());
    String confFileName = JobHistoryUtils
        .getIntermediateConfFileName(jobIndexInfo.getJobId());
    String summaryFileName = JobHistoryUtils
        .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
    HistoryFileInfo fileInfo = createHistoryFileInfo(fs.getPath(), new Path(fs
        .getPath().getParent(), confFileName), new Path(fs.getPath()
        .getParent(), summaryFileName), jobIndexInfo, true);
    jobListCache.addIfAbsent(fileInfo);
  }
}
 
Example #21
Source File: MRAppMaster.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static FSDataInputStream getPreviousJobHistoryStream(
    Configuration conf, ApplicationAttemptId appAttemptId)
    throws IOException {
  Path historyFile = JobHistoryUtils.getPreviousJobHistoryPath(conf,
      appAttemptId);
  LOG.info("Previous history file is at " + historyFile);
  return historyFile.getFileSystem(conf).open(historyFile);
}
 
Example #22
Source File: HistoryFileManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  this.conf = conf;

  int serialNumberLowDigits = 3;
  serialNumberFormat = ("%0"
      + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits)
      + "d");

  long maxFSWaitTime = conf.getLong(
      JHAdminConfig.MR_HISTORY_MAX_START_WAIT_TIME,
      JHAdminConfig.DEFAULT_MR_HISTORY_MAX_START_WAIT_TIME);
  createHistoryDirs(new SystemClock(), 10 * 1000, maxFSWaitTime);

  this.aclsMgr = new JobACLsManager(conf);

  maxHistoryAge = conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS,
      JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);
  
  jobListCache = createJobListCache();

  serialNumberIndex = new SerialNumberIndex(conf.getInt(
      JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE,
      JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE));

  int numMoveThreads = conf.getInt(
      JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT,
      JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT);
  ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat(
      "MoveIntermediateToDone Thread #%d").build();
  moveToDoneExecutor = new ThreadPoolExecutor(numMoveThreads, numMoveThreads,
      1, TimeUnit.HOURS, new LinkedBlockingQueue<Runnable>(), tf);

  super.serviceInit(conf);
}
 
Example #23
Source File: HistoryFileManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  this.conf = conf;

  int serialNumberLowDigits = 3;
  serialNumberFormat = ("%0"
      + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits)
      + "d");

  long maxFSWaitTime = conf.getLong(
      JHAdminConfig.MR_HISTORY_MAX_START_WAIT_TIME,
      JHAdminConfig.DEFAULT_MR_HISTORY_MAX_START_WAIT_TIME);
  createHistoryDirs(new SystemClock(), 10 * 1000, maxFSWaitTime);

  this.aclsMgr = new JobACLsManager(conf);

  maxHistoryAge = conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS,
      JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);
  
  jobListCache = createJobListCache();

  serialNumberIndex = new SerialNumberIndex(conf.getInt(
      JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE,
      JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE));

  int numMoveThreads = conf.getInt(
      JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT,
      JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT);
  moveToDoneExecutor = createMoveToDoneThreadPool(numMoveThreads);
  super.serviceInit(conf);
}
 
Example #24
Source File: MRAppMaster.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static FSDataInputStream getPreviousJobHistoryStream(
    Configuration conf, ApplicationAttemptId appAttemptId)
    throws IOException {
  Path historyFile = JobHistoryUtils.getPreviousJobHistoryPath(conf,
      appAttemptId);
  LOG.info("Previous history file is at " + historyFile);
  return historyFile.getFileSystem(conf).open(historyFile);
}
 
Example #25
Source File: HistoryFileManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
private Path canonicalHistoryLogPath(JobId id, String timestampComponent) {
  return new Path(doneDirPrefixPath, JobHistoryUtils.historyLogSubdirectory(
      id, timestampComponent, serialNumberFormat));
}
 
Example #26
Source File: HistoryFileManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
private Path canonicalHistoryLogPath(JobId id, long millisecondTime) {
  String timestampComponent = JobHistoryUtils
      .timestampDirectoryComponent(millisecondTime);
  return new Path(doneDirPrefixPath, JobHistoryUtils.historyLogSubdirectory(
      id, timestampComponent, serialNumberFormat));
}
 
Example #27
Source File: MiniTezCluster.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
@Override
public void serviceInit(Configuration conf) throws Exception {
  conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_TEZ_FRAMEWORK_NAME);
  // blacklisting disabled to prevent scheduling issues
  conf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
  if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) {
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(getTestWorkDir(),
        "apps_staging_dir" + Path.SEPARATOR).getAbsolutePath());
  }
  
  if (conf.get(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC) == null) {
    // nothing defined. set quick delete value
    conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 0l);
  }
  
  File appJarLocalFile = new File(MiniTezCluster.APPJAR);

  if (!appJarLocalFile.exists()) {
    String message = "TezAppJar " + MiniTezCluster.APPJAR
        + " not found. Exiting.";
    LOG.info(message);
    throw new TezUncheckedException(message);
  }
  
  FileSystem fs = FileSystem.get(conf);
  Path testRootDir = fs.makeQualified(new Path("target", getName() + "-tmpDir"));
  Path appRemoteJar = new Path(testRootDir, "TezAppJar.jar");
  // Copy AppJar and make it public.
  Path appMasterJar = new Path(MiniTezCluster.APPJAR);
  fs.copyFromLocalFile(appMasterJar, appRemoteJar);
  fs.setPermission(appRemoteJar, new FsPermission("777"));

  conf.set(TezConfiguration.TEZ_LIB_URIS, appRemoteJar.toUri().toString());
  LOG.info("Set TEZ-LIB-URI to: " + conf.get(TezConfiguration.TEZ_LIB_URIS));

  // VMEM monitoring disabled, PMEM monitoring enabled.
  conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
  conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);

  conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,  "000");

  try {
    Path stagingPath = FileContext.getFileContext(conf).makeQualified(
        new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR)));
    /*
     * Re-configure the staging path on Windows if the file system is localFs.
     * We need to use a absolute path that contains the drive letter. The unit
     * test could run on a different drive than the AM. We can run into the
     * issue that job files are localized to the drive where the test runs on,
     * while the AM starts on a different drive and fails to find the job
     * metafiles. Using absolute path can avoid this ambiguity.
     */
    if (Path.WINDOWS) {
      if (LocalFileSystem.class.isInstance(stagingPath.getFileSystem(conf))) {
        conf.set(MRJobConfig.MR_AM_STAGING_DIR,
            new File(conf.get(MRJobConfig.MR_AM_STAGING_DIR))
                .getAbsolutePath());
      }
    }
    FileContext fc=FileContext.getFileContext(stagingPath.toUri(), conf);
    if (fc.util().exists(stagingPath)) {
      LOG.info(stagingPath + " exists! deleting...");
      fc.delete(stagingPath, true);
    }
    LOG.info("mkdir: " + stagingPath);
    fc.mkdir(stagingPath, null, true);

    //mkdir done directory as well
    String doneDir =
        JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
    Path doneDirPath = fc.makeQualified(new Path(doneDir));
    fc.mkdir(doneDirPath, null, true);
  } catch (IOException e) {
    throw new TezUncheckedException("Could not create staging directory. ", e);
  }
  conf.set(MRConfig.MASTER_ADDRESS, "test");

  //configure the shuffle service in NM
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
      new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT,
      ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class,
      Service.class);

  // Non-standard shuffle port
  conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);

  conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR,
      DefaultContainerExecutor.class, ContainerExecutor.class);

  // TestMRJobs is for testing non-uberized operation only; see TestUberAM
  // for corresponding uberized tests.
  conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
  super.serviceInit(conf);
}
 
Example #28
Source File: YARNRunner.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
@Override
public String getJobHistoryDir() throws IOException, InterruptedException {
  return JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
}
 
Example #29
Source File: HistoryFileManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
List<FileStatus> getHistoryDirsForCleaning(long cutoff) throws IOException {
    return JobHistoryUtils.
      getHistoryDirsForCleaning(doneDirFc, doneDirPrefixPath, cutoff);
}
 
Example #30
Source File: HistoryFileManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Clean up older history files.
 * 
 * @throws IOException
 *           on any error trying to remove the entries.
 */
@SuppressWarnings("unchecked")
void clean() throws IOException {
  long cutoff = System.currentTimeMillis() - maxHistoryAge;
  boolean halted = false;
  List<FileStatus> serialDirList = getHistoryDirsForCleaning(cutoff);
  // Sort in ascending order. Relies on YYYY/MM/DD/Serial
  Collections.sort(serialDirList);
  for (FileStatus serialDir : serialDirList) {
    List<FileStatus> historyFileList = scanDirectoryForHistoryFiles(
        serialDir.getPath(), doneDirFc);
    for (FileStatus historyFile : historyFileList) {
      JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(historyFile
          .getPath().getName());
      long effectiveTimestamp = getEffectiveTimestamp(
          jobIndexInfo.getFinishTime(), historyFile);
      if (effectiveTimestamp <= cutoff) {
        HistoryFileInfo fileInfo = this.jobListCache.get(jobIndexInfo
            .getJobId());
        if (fileInfo == null) {
          String confFileName = JobHistoryUtils
              .getIntermediateConfFileName(jobIndexInfo.getJobId());

          fileInfo = new HistoryFileInfo(historyFile.getPath(), new Path(
              historyFile.getPath().getParent(), confFileName), null,
              jobIndexInfo, true);
        }
        deleteJobFromDone(fileInfo);
      } else {
        halted = true;
        break;
      }
    }
    if (!halted) {
      deleteDir(serialDir);
      removeDirectoryFromSerialNumberIndex(serialDir.getPath());
      existingDoneSubdirs.remove(serialDir.getPath());
    } else {
      break; // Don't scan any more directories.
    }
  }
}