Java Code Examples for org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils

The following examples show how to use org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: JobHistoryEventHandler.java    License: Apache License 2.0 6 votes vote down vote up
private void moveToDoneNow(Path fromPath, Path toPath) throws IOException {
  // check if path exists, in case of retries it may not exist
  if (stagingDirFS.exists(fromPath)) {
    LOG.info("Copying " + fromPath.toString() + " to " + toPath.toString());
    // TODO temporarily removing the existing dst
    if (doneDirFS.exists(toPath)) {
      doneDirFS.delete(toPath, true);
    }
    boolean copied = FileUtil.copy(stagingDirFS, fromPath, doneDirFS, toPath,
        false, getConfig());

    if (copied)
      LOG.info("Copied to done location: " + toPath);
    else 
      LOG.info("copy failed");
    doneDirFS.setPermission(toPath, new FsPermission(
        JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS));
  }
}
 
Example 2
Source Project: hadoop   Source File: JobHistoryCopyService.java    License: Apache License 2.0 6 votes vote down vote up
public static FSDataInputStream getPreviousJobHistoryFileStream(
    Configuration conf, ApplicationAttemptId applicationAttemptId)
    throws IOException {
  FSDataInputStream in = null;
  Path historyFile = null;
  String jobId =
      TypeConverter.fromYarn(applicationAttemptId.getApplicationId())
        .toString();
  String jobhistoryDir =
      JobHistoryUtils.getConfiguredHistoryStagingDirPrefix(conf, jobId);
  Path histDirPath =
      FileContext.getFileContext(conf).makeQualified(new Path(jobhistoryDir));
  FileContext fc = FileContext.getFileContext(histDirPath.toUri(), conf);
  // read the previous history file
  historyFile =
      fc.makeQualified(JobHistoryUtils.getStagingJobHistoryFile(histDirPath,
        jobId, (applicationAttemptId.getAttemptId() - 1)));
  LOG.info("History file is at " + historyFile);
  in = fc.open(historyFile);
  return in;
}
 
Example 3
Source Project: hadoop   Source File: TestJobHistoryEventHandler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetHistoryIntermediateDoneDirForUser() throws IOException {
  // Test relative path
  Configuration conf = new Configuration();
  conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR,
      "/mapred/history/done_intermediate");
  conf.set(MRJobConfig.USER_NAME, System.getProperty("user.name"));
  String pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
  Assert.assertEquals("/mapred/history/done_intermediate/" +
      System.getProperty("user.name"), pathStr);

  // Test fully qualified path
  // Create default configuration pointing to the minicluster
  conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
      dfsCluster.getURI().toString());
  FileOutputStream os = new FileOutputStream(coreSitePath);
  conf.writeXml(os);
  os.close();
  // Simulate execution under a non-default namenode
  conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
          "file:///");
  pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
  Assert.assertEquals(dfsCluster.getURI().toString() +
      "/mapred/history/done_intermediate/" + System.getProperty("user.name"),
      pathStr);
}
 
Example 4
Source Project: hadoop   Source File: HistoryFileManager.java    License: Apache License 2.0 6 votes vote down vote up
private void removeDirectoryFromSerialNumberIndex(Path serialDirPath) {
  String serialPart = serialDirPath.getName();
  String timeStampPart = JobHistoryUtils
      .getTimestampPartFromPath(serialDirPath.toString());
  if (timeStampPart == null) {
    LOG.warn("Could not find timestamp portion from path: "
        + serialDirPath.toString() + ". Continuing with next");
    return;
  }
  if (serialPart == null) {
    LOG.warn("Could not find serial portion from path: "
        + serialDirPath.toString() + ". Continuing with next");
    return;
  }
  serialNumberIndex.remove(serialPart, timeStampPart);
}
 
Example 5
Source Project: hadoop   Source File: HistoryFileManager.java    License: Apache License 2.0 6 votes vote down vote up
private void addDirectoryToSerialNumberIndex(Path serialDirPath) {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Adding " + serialDirPath + " to serial index");
  }
  String serialPart = serialDirPath.getName();
  String timestampPart = JobHistoryUtils
      .getTimestampPartFromPath(serialDirPath.toString());
  if (timestampPart == null) {
    LOG.warn("Could not find timestamp portion from path: " + serialDirPath
        + ". Continuing with next");
    return;
  }
  if (serialPart == null) {
    LOG.warn("Could not find serial portion from path: "
        + serialDirPath.toString() + ". Continuing with next");
  } else {
    serialNumberIndex.add(serialPart, timestampPart);
  }
}
 
Example 6
Source Project: hadoop   Source File: HistoryFileManager.java    License: Apache License 2.0 6 votes vote down vote up
private void addDirectoryToJobListCache(Path path) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Adding " + path + " to job list cache.");
  }
  List<FileStatus> historyFileList = scanDirectoryForHistoryFiles(path,
      doneDirFc);
  for (FileStatus fs : historyFileList) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Adding in history for " + fs.getPath());
    }
    JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
        .getName());
    String confFileName = JobHistoryUtils
        .getIntermediateConfFileName(jobIndexInfo.getJobId());
    String summaryFileName = JobHistoryUtils
        .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
    HistoryFileInfo fileInfo = createHistoryFileInfo(fs.getPath(), new Path(fs
        .getPath().getParent(), confFileName), new Path(fs.getPath()
        .getParent(), summaryFileName), jobIndexInfo, true);
    jobListCache.addIfAbsent(fileInfo);
  }
}
 
Example 7
Source Project: hadoop   Source File: HistoryFileManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Scans the intermediate directory to find user directories. Scans these for
 * history files if the modification time for the directory has changed. Once
 * it finds history files it starts the process of moving them to the done 
 * directory.
 * 
 * @throws IOException
 *           if there was a error while scanning
 */
void scanIntermediateDirectory() throws IOException {
  // TODO it would be great to limit how often this happens, except in the
  // case where we are looking for a particular job.
  List<FileStatus> userDirList = JobHistoryUtils.localGlobber(
      intermediateDoneDirFc, intermediateDoneDirPath, "");
  LOG.debug("Scanning intermediate dirs");
  for (FileStatus userDir : userDirList) {
    String name = userDir.getPath().getName();
    UserLogDir dir = userDirModificationTimeMap.get(name);
    if(dir == null) {
      dir = new UserLogDir();
      UserLogDir old = userDirModificationTimeMap.putIfAbsent(name, dir);
      if(old != null) {
        dir = old;
      }
    }
    dir.scanIfNeeded(userDir);
  }
}
 
Example 8
Source Project: hadoop   Source File: HistoryFileManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Searches the job history file FileStatus list for the specified JobId.
 * 
 * @param fileStatusList
 *          fileStatus list of Job History Files.
 * @param jobId
 *          The JobId to find.
 * @return A FileInfo object for the jobId, null if not found.
 * @throws IOException
 */
private HistoryFileInfo getJobFileInfo(List<FileStatus> fileStatusList,
    JobId jobId) throws IOException {
  for (FileStatus fs : fileStatusList) {
    JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
        .getName());
    if (jobIndexInfo.getJobId().equals(jobId)) {
      String confFileName = JobHistoryUtils
          .getIntermediateConfFileName(jobIndexInfo.getJobId());
      String summaryFileName = JobHistoryUtils
          .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
      HistoryFileInfo fileInfo = createHistoryFileInfo(fs.getPath(), new Path(
          fs.getPath().getParent(), confFileName), new Path(fs.getPath()
          .getParent(), summaryFileName), jobIndexInfo, true);
      return fileInfo;
    }
  }
  return null;
}
 
Example 9
Source Project: hadoop   Source File: HistoryFileManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Scans old directories known by the idToDateString map for the specified
 * jobId. If the number of directories is higher than the supported size of
 * the idToDateString cache, the jobId will not be found.
 * 
 * @param jobId
 *          the jobId.
 * @return
 * @throws IOException
 */
private HistoryFileInfo scanOldDirsForJob(JobId jobId) throws IOException {
  String boxedSerialNumber = JobHistoryUtils.serialNumberDirectoryComponent(
      jobId, serialNumberFormat);
  Set<String> dateStringSet = serialNumberIndex.get(boxedSerialNumber);
  if (dateStringSet == null) {
    return null;
  }
  for (String timestampPart : dateStringSet) {
    Path logDir = canonicalHistoryLogPath(jobId, timestampPart);
    List<FileStatus> fileStatusList = scanDirectoryForHistoryFiles(logDir,
        doneDirFc);
    HistoryFileInfo fileInfo = getJobFileInfo(fileStatusList, jobId);
    if (fileInfo != null) {
      return fileInfo;
    }
  }
  return null;
}
 
Example 10
Source Project: hadoop   Source File: HistoryFileManager.java    License: Apache License 2.0 6 votes vote down vote up
private void makeDoneSubdir(Path path) throws IOException {
  try {
    doneDirFc.getFileStatus(path);
    existingDoneSubdirs.add(path);
  } catch (FileNotFoundException fnfE) {
    try {
      FsPermission fsp = new FsPermission(
          JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION);
      doneDirFc.mkdir(path, fsp, true);
      FileStatus fsStatus = doneDirFc.getFileStatus(path);
      LOG.info("Perms after creating " + fsStatus.getPermission().toShort()
          + ", Expected: " + fsp.toShort());
      if (fsStatus.getPermission().toShort() != fsp.toShort()) {
        LOG.info("Explicitly setting permissions to : " + fsp.toShort()
            + ", " + fsp);
        doneDirFc.setPermission(path, fsp);
      }
      existingDoneSubdirs.add(path);
    } catch (FileAlreadyExistsException faeE) { // Nothing to do.
    }
  }
}
 
Example 11
Source Project: big-c   Source File: JobHistoryEventHandler.java    License: Apache License 2.0 6 votes vote down vote up
private void moveToDoneNow(Path fromPath, Path toPath) throws IOException {
  // check if path exists, in case of retries it may not exist
  if (stagingDirFS.exists(fromPath)) {
    LOG.info("Copying " + fromPath.toString() + " to " + toPath.toString());
    // TODO temporarily removing the existing dst
    if (doneDirFS.exists(toPath)) {
      doneDirFS.delete(toPath, true);
    }
    boolean copied = FileUtil.copy(stagingDirFS, fromPath, doneDirFS, toPath,
        false, getConfig());

    if (copied)
      LOG.info("Copied to done location: " + toPath);
    else 
      LOG.info("copy failed");
    doneDirFS.setPermission(toPath, new FsPermission(
        JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS));
  }
}
 
Example 12
Source Project: big-c   Source File: JobHistoryCopyService.java    License: Apache License 2.0 6 votes vote down vote up
public static FSDataInputStream getPreviousJobHistoryFileStream(
    Configuration conf, ApplicationAttemptId applicationAttemptId)
    throws IOException {
  FSDataInputStream in = null;
  Path historyFile = null;
  String jobId =
      TypeConverter.fromYarn(applicationAttemptId.getApplicationId())
        .toString();
  String jobhistoryDir =
      JobHistoryUtils.getConfiguredHistoryStagingDirPrefix(conf, jobId);
  Path histDirPath =
      FileContext.getFileContext(conf).makeQualified(new Path(jobhistoryDir));
  FileContext fc = FileContext.getFileContext(histDirPath.toUri(), conf);
  // read the previous history file
  historyFile =
      fc.makeQualified(JobHistoryUtils.getStagingJobHistoryFile(histDirPath,
        jobId, (applicationAttemptId.getAttemptId() - 1)));
  LOG.info("History file is at " + historyFile);
  in = fc.open(historyFile);
  return in;
}
 
Example 13
Source Project: big-c   Source File: TestJobHistoryEventHandler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetHistoryIntermediateDoneDirForUser() throws IOException {
  // Test relative path
  Configuration conf = new Configuration();
  conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR,
      "/mapred/history/done_intermediate");
  conf.set(MRJobConfig.USER_NAME, System.getProperty("user.name"));
  String pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
  Assert.assertEquals("/mapred/history/done_intermediate/" +
      System.getProperty("user.name"), pathStr);

  // Test fully qualified path
  // Create default configuration pointing to the minicluster
  conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
      dfsCluster.getURI().toString());
  FileOutputStream os = new FileOutputStream(coreSitePath);
  conf.writeXml(os);
  os.close();
  // Simulate execution under a non-default namenode
  conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
          "file:///");
  pathStr = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
  Assert.assertEquals(dfsCluster.getURI().toString() +
      "/mapred/history/done_intermediate/" + System.getProperty("user.name"),
      pathStr);
}
 
Example 14
Source Project: big-c   Source File: HistoryFileManager.java    License: Apache License 2.0 6 votes vote down vote up
private void removeDirectoryFromSerialNumberIndex(Path serialDirPath) {
  String serialPart = serialDirPath.getName();
  String timeStampPart = JobHistoryUtils
      .getTimestampPartFromPath(serialDirPath.toString());
  if (timeStampPart == null) {
    LOG.warn("Could not find timestamp portion from path: "
        + serialDirPath.toString() + ". Continuing with next");
    return;
  }
  if (serialPart == null) {
    LOG.warn("Could not find serial portion from path: "
        + serialDirPath.toString() + ". Continuing with next");
    return;
  }
  serialNumberIndex.remove(serialPart, timeStampPart);
}
 
Example 15
Source Project: big-c   Source File: HistoryFileManager.java    License: Apache License 2.0 6 votes vote down vote up
private void addDirectoryToSerialNumberIndex(Path serialDirPath) {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Adding " + serialDirPath + " to serial index");
  }
  String serialPart = serialDirPath.getName();
  String timestampPart = JobHistoryUtils
      .getTimestampPartFromPath(serialDirPath.toString());
  if (timestampPart == null) {
    LOG.warn("Could not find timestamp portion from path: " + serialDirPath
        + ". Continuing with next");
    return;
  }
  if (serialPart == null) {
    LOG.warn("Could not find serial portion from path: "
        + serialDirPath.toString() + ". Continuing with next");
  } else {
    serialNumberIndex.add(serialPart, timestampPart);
  }
}
 
Example 16
Source Project: big-c   Source File: HistoryFileManager.java    License: Apache License 2.0 6 votes vote down vote up
private void addDirectoryToJobListCache(Path path) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Adding " + path + " to job list cache.");
  }
  List<FileStatus> historyFileList = scanDirectoryForHistoryFiles(path,
      doneDirFc);
  for (FileStatus fs : historyFileList) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Adding in history for " + fs.getPath());
    }
    JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
        .getName());
    String confFileName = JobHistoryUtils
        .getIntermediateConfFileName(jobIndexInfo.getJobId());
    String summaryFileName = JobHistoryUtils
        .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
    HistoryFileInfo fileInfo = new HistoryFileInfo(fs.getPath(), new Path(fs
        .getPath().getParent(), confFileName), new Path(fs.getPath()
        .getParent(), summaryFileName), jobIndexInfo, true);
    jobListCache.addIfAbsent(fileInfo);
  }
}
 
Example 17
Source Project: big-c   Source File: HistoryFileManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Scans the intermediate directory to find user directories. Scans these for
 * history files if the modification time for the directory has changed. Once
 * it finds history files it starts the process of moving them to the done 
 * directory.
 * 
 * @throws IOException
 *           if there was a error while scanning
 */
void scanIntermediateDirectory() throws IOException {
  // TODO it would be great to limit how often this happens, except in the
  // case where we are looking for a particular job.
  List<FileStatus> userDirList = JobHistoryUtils.localGlobber(
      intermediateDoneDirFc, intermediateDoneDirPath, "");
  LOG.debug("Scanning intermediate dirs");
  for (FileStatus userDir : userDirList) {
    String name = userDir.getPath().getName();
    UserLogDir dir = userDirModificationTimeMap.get(name);
    if(dir == null) {
      dir = new UserLogDir();
      UserLogDir old = userDirModificationTimeMap.putIfAbsent(name, dir);
      if(old != null) {
        dir = old;
      }
    }
    dir.scanIfNeeded(userDir);
  }
}
 
Example 18
Source Project: big-c   Source File: HistoryFileManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Searches the job history file FileStatus list for the specified JobId.
 * 
 * @param fileStatusList
 *          fileStatus list of Job History Files.
 * @param jobId
 *          The JobId to find.
 * @return A FileInfo object for the jobId, null if not found.
 * @throws IOException
 */
private HistoryFileInfo getJobFileInfo(List<FileStatus> fileStatusList,
    JobId jobId) throws IOException {
  for (FileStatus fs : fileStatusList) {
    JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath()
        .getName());
    if (jobIndexInfo.getJobId().equals(jobId)) {
      String confFileName = JobHistoryUtils
          .getIntermediateConfFileName(jobIndexInfo.getJobId());
      String summaryFileName = JobHistoryUtils
          .getIntermediateSummaryFileName(jobIndexInfo.getJobId());
      HistoryFileInfo fileInfo = new HistoryFileInfo(fs.getPath(), new Path(
          fs.getPath().getParent(), confFileName), new Path(fs.getPath()
          .getParent(), summaryFileName), jobIndexInfo, true);
      return fileInfo;
    }
  }
  return null;
}
 
Example 19
Source Project: big-c   Source File: HistoryFileManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Scans old directories known by the idToDateString map for the specified
 * jobId. If the number of directories is higher than the supported size of
 * the idToDateString cache, the jobId will not be found.
 * 
 * @param jobId
 *          the jobId.
 * @return
 * @throws IOException
 */
private HistoryFileInfo scanOldDirsForJob(JobId jobId) throws IOException {
  String boxedSerialNumber = JobHistoryUtils.serialNumberDirectoryComponent(
      jobId, serialNumberFormat);
  Set<String> dateStringSet = serialNumberIndex.get(boxedSerialNumber);
  if (dateStringSet == null) {
    return null;
  }
  for (String timestampPart : dateStringSet) {
    Path logDir = canonicalHistoryLogPath(jobId, timestampPart);
    List<FileStatus> fileStatusList = scanDirectoryForHistoryFiles(logDir,
        doneDirFc);
    HistoryFileInfo fileInfo = getJobFileInfo(fileStatusList, jobId);
    if (fileInfo != null) {
      return fileInfo;
    }
  }
  return null;
}
 
Example 20
Source Project: big-c   Source File: HistoryFileManager.java    License: Apache License 2.0 6 votes vote down vote up
private void makeDoneSubdir(Path path) throws IOException {
  try {
    doneDirFc.getFileStatus(path);
    existingDoneSubdirs.add(path);
  } catch (FileNotFoundException fnfE) {
    try {
      FsPermission fsp = new FsPermission(
          JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION);
      doneDirFc.mkdir(path, fsp, true);
      FileStatus fsStatus = doneDirFc.getFileStatus(path);
      LOG.info("Perms after creating " + fsStatus.getPermission().toShort()
          + ", Expected: " + fsp.toShort());
      if (fsStatus.getPermission().toShort() != fsp.toShort()) {
        LOG.info("Explicitly setting permissions to : " + fsp.toShort()
            + ", " + fsp);
        doneDirFc.setPermission(path, fsp);
      }
      existingDoneSubdirs.add(path);
    } catch (FileAlreadyExistsException faeE) { // Nothing to do.
    }
  }
}
 
Example 21
Source Project: hadoop   Source File: MRAppMaster.java    License: Apache License 2.0 5 votes vote down vote up
private static FSDataInputStream getPreviousJobHistoryStream(
    Configuration conf, ApplicationAttemptId appAttemptId)
    throws IOException {
  Path historyFile = JobHistoryUtils.getPreviousJobHistoryPath(conf,
      appAttemptId);
  LOG.info("Previous history file is at " + historyFile);
  return historyFile.getFileSystem(conf).open(historyFile);
}
 
Example 22
Source Project: hadoop   Source File: HistoryFileManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  this.conf = conf;

  int serialNumberLowDigits = 3;
  serialNumberFormat = ("%0"
      + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits)
      + "d");

  long maxFSWaitTime = conf.getLong(
      JHAdminConfig.MR_HISTORY_MAX_START_WAIT_TIME,
      JHAdminConfig.DEFAULT_MR_HISTORY_MAX_START_WAIT_TIME);
  createHistoryDirs(new SystemClock(), 10 * 1000, maxFSWaitTime);

  this.aclsMgr = new JobACLsManager(conf);

  maxHistoryAge = conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS,
      JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);
  
  jobListCache = createJobListCache();

  serialNumberIndex = new SerialNumberIndex(conf.getInt(
      JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE,
      JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE));

  int numMoveThreads = conf.getInt(
      JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT,
      JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT);
  moveToDoneExecutor = createMoveToDoneThreadPool(numMoveThreads);
  super.serviceInit(conf);
}
 
Example 23
Source Project: big-c   Source File: MRAppMaster.java    License: Apache License 2.0 5 votes vote down vote up
private static FSDataInputStream getPreviousJobHistoryStream(
    Configuration conf, ApplicationAttemptId appAttemptId)
    throws IOException {
  Path historyFile = JobHistoryUtils.getPreviousJobHistoryPath(conf,
      appAttemptId);
  LOG.info("Previous history file is at " + historyFile);
  return historyFile.getFileSystem(conf).open(historyFile);
}
 
Example 24
Source Project: big-c   Source File: HistoryFileManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  this.conf = conf;

  int serialNumberLowDigits = 3;
  serialNumberFormat = ("%0"
      + (JobHistoryUtils.SERIAL_NUMBER_DIRECTORY_DIGITS + serialNumberLowDigits)
      + "d");

  long maxFSWaitTime = conf.getLong(
      JHAdminConfig.MR_HISTORY_MAX_START_WAIT_TIME,
      JHAdminConfig.DEFAULT_MR_HISTORY_MAX_START_WAIT_TIME);
  createHistoryDirs(new SystemClock(), 10 * 1000, maxFSWaitTime);

  this.aclsMgr = new JobACLsManager(conf);

  maxHistoryAge = conf.getLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS,
      JHAdminConfig.DEFAULT_MR_HISTORY_MAX_AGE);
  
  jobListCache = createJobListCache();

  serialNumberIndex = new SerialNumberIndex(conf.getInt(
      JHAdminConfig.MR_HISTORY_DATESTRING_CACHE_SIZE,
      JHAdminConfig.DEFAULT_MR_HISTORY_DATESTRING_CACHE_SIZE));

  int numMoveThreads = conf.getInt(
      JHAdminConfig.MR_HISTORY_MOVE_THREAD_COUNT,
      JHAdminConfig.DEFAULT_MR_HISTORY_MOVE_THREAD_COUNT);
  ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat(
      "MoveIntermediateToDone Thread #%d").build();
  moveToDoneExecutor = new ThreadPoolExecutor(numMoveThreads, numMoveThreads,
      1, TimeUnit.HOURS, new LinkedBlockingQueue<Runnable>(), tf);

  super.serviceInit(conf);
}
 
Example 25
Source Project: hadoop   Source File: JobHistoryEventHandler.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Create an event writer for the Job represented by the jobID.
 * Writes out the job configuration to the log directory.
 * This should be the first call to history for a job
 * 
 * @param jobId the jobId.
 * @param amStartedEvent
 * @throws IOException
 */
protected void setupEventWriter(JobId jobId, AMStartedEvent amStartedEvent)
    throws IOException {
  if (stagingDirPath == null) {
    LOG.error("Log Directory is null, returning");
    throw new IOException("Missing Log Directory for History");
  }

  MetaInfo oldFi = fileMap.get(jobId);
  Configuration conf = getConfig();

  // TODO Ideally this should be written out to the job dir
  // (.staging/jobid/files - RecoveryService will need to be patched)
  Path historyFile = JobHistoryUtils.getStagingJobHistoryFile(
      stagingDirPath, jobId, startCount);
  String user = UserGroupInformation.getCurrentUser().getShortUserName();
  if (user == null) {
    throw new IOException(
        "User is null while setting up jobhistory eventwriter");
  }

  String jobName = context.getJob(jobId).getName();
  EventWriter writer = (oldFi == null) ? null : oldFi.writer;
 
  Path logDirConfPath =
      JobHistoryUtils.getStagingConfFile(stagingDirPath, jobId, startCount);
  if (writer == null) {
    try {
      writer = createEventWriter(historyFile);
      LOG.info("Event Writer setup for JobId: " + jobId + ", File: "
          + historyFile);
    } catch (IOException ioe) {
      LOG.info("Could not create log file: [" + historyFile + "] + for job "
          + "[" + jobName + "]");
      throw ioe;
    }
    
    //Write out conf only if the writer isn't already setup.
    if (conf != null) {
      // TODO Ideally this should be written out to the job dir
      // (.staging/jobid/files - RecoveryService will need to be patched)
      FSDataOutputStream jobFileOut = null;
      try {
        if (logDirConfPath != null) {
          jobFileOut = stagingDirFS.create(logDirConfPath, true);
          conf.writeXml(jobFileOut);
          jobFileOut.close();
        }
      } catch (IOException e) {
        LOG.info("Failed to write the job configuration file", e);
        throw e;
      }
    }
  }

  String queueName = JobConf.DEFAULT_QUEUE_NAME;
  if (conf != null) {
    queueName = conf.get(MRJobConfig.QUEUE_NAME, JobConf.DEFAULT_QUEUE_NAME);
  }

  MetaInfo fi = new MetaInfo(historyFile, logDirConfPath, writer,
      user, jobName, jobId, amStartedEvent.getForcedJobStateOnShutDown(),
      queueName);
  fi.getJobSummary().setJobId(jobId);
  fi.getJobSummary().setJobLaunchTime(amStartedEvent.getStartTime());
  fi.getJobSummary().setJobSubmitTime(amStartedEvent.getSubmitTime());
  fi.getJobIndexInfo().setJobStartTime(amStartedEvent.getStartTime());
  fi.getJobIndexInfo().setSubmitTime(amStartedEvent.getSubmitTime());
  fileMap.put(jobId, fi);
}
 
Example 26
Source Project: hadoop   Source File: YARNRunner.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public String getJobHistoryDir() throws IOException, InterruptedException {
  return JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
}
 
Example 27
Source Project: hadoop   Source File: MiniMRYarnCluster.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void serviceInit(Configuration conf) throws Exception {
  conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
  if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) {
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(getTestWorkDir(),
        "apps_staging_dir/").getAbsolutePath());
  }

  // By default, VMEM monitoring disabled, PMEM monitoring enabled.
  if (!conf.getBoolean(
      MRConfig.MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING,
      MRConfig.DEFAULT_MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING)) {
    conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
    conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
  }

  conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,  "000");

  try {
    Path stagingPath = FileContext.getFileContext(conf).makeQualified(
        new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR)));
    /*
     * Re-configure the staging path on Windows if the file system is localFs.
     * We need to use a absolute path that contains the drive letter. The unit
     * test could run on a different drive than the AM. We can run into the
     * issue that job files are localized to the drive where the test runs on,
     * while the AM starts on a different drive and fails to find the job
     * metafiles. Using absolute path can avoid this ambiguity.
     */
    if (Path.WINDOWS) {
      if (LocalFileSystem.class.isInstance(stagingPath.getFileSystem(conf))) {
        conf.set(MRJobConfig.MR_AM_STAGING_DIR,
            new File(conf.get(MRJobConfig.MR_AM_STAGING_DIR))
                .getAbsolutePath());
      }
    }
    FileContext fc=FileContext.getFileContext(stagingPath.toUri(), conf);
    if (fc.util().exists(stagingPath)) {
      LOG.info(stagingPath + " exists! deleting...");
      fc.delete(stagingPath, true);
    }
    LOG.info("mkdir: " + stagingPath);
    //mkdir the staging directory so that right permissions are set while running as proxy user
    fc.mkdir(stagingPath, null, true);
    //mkdir done directory as well 
    String doneDir = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
    Path doneDirPath = fc.makeQualified(new Path(doneDir));
    fc.mkdir(doneDirPath, null, true);
  } catch (IOException e) {
    throw new YarnRuntimeException("Could not create staging directory. ", e);
  }
  conf.set(MRConfig.MASTER_ADDRESS, "test"); // The default is local because of
                                           // which shuffle doesn't happen
  //configure the shuffle service in NM
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
      new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT,
      ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class,
      Service.class);

  // Non-standard shuffle port
  conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);

  conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR,
      DefaultContainerExecutor.class, ContainerExecutor.class);

  // TestMRJobs is for testing non-uberized operation only; see TestUberAM
  // for corresponding uberized tests.
  conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);

  super.serviceInit(conf);
}
 
Example 28
Source Project: hadoop   Source File: HistoryFileManager.java    License: Apache License 2.0 4 votes vote down vote up
protected List<FileStatus> scanDirectoryForHistoryFiles(Path path,
    FileContext fc) throws IOException {
  return scanDirectory(path, fc, JobHistoryUtils.getHistoryFileFilter());
}
 
Example 29
Source Project: hadoop   Source File: HistoryFileManager.java    License: Apache License 2.0 4 votes vote down vote up
private Path canonicalHistoryLogPath(JobId id, String timestampComponent) {
  return new Path(doneDirPrefixPath, JobHistoryUtils.historyLogSubdirectory(
      id, timestampComponent, serialNumberFormat));
}
 
Example 30
Source Project: hadoop   Source File: HistoryFileManager.java    License: Apache License 2.0 4 votes vote down vote up
private Path canonicalHistoryLogPath(JobId id, long millisecondTime) {
  String timestampComponent = JobHistoryUtils
      .timestampDirectoryComponent(millisecondTime);
  return new Path(doneDirPrefixPath, JobHistoryUtils.historyLogSubdirectory(
      id, timestampComponent, serialNumberFormat));
}