org.apache.hadoop.mapred.JobHistory.JobInfo Java Examples

The following examples show how to use org.apache.hadoop.mapred.JobHistory.JobInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JSPUtil.java    From RDFS with Apache License 2.0 6 votes vote down vote up
public static JobInfo getJobInfo(HttpServletRequest request, FileSystem fs)
    throws IOException {
  String jobid = request.getParameter("jobid");
  String logFile = request.getParameter("logFile");
  synchronized(jobHistoryCache) {
    JobInfo jobInfo = jobHistoryCache.remove(jobid);
    if (jobInfo == null) {
      jobInfo = new JobHistory.JobInfo(jobid);
      LOG.info("Loading Job History file "+jobid + ".   Cache size is " +
          jobHistoryCache.size());
      DefaultJobHistoryParser.parseJobTasks( logFile, jobInfo, fs) ;
    }
    jobHistoryCache.put(jobid, jobInfo);
    if (jobHistoryCache.size() > CACHE_SIZE) {
      Iterator<Map.Entry<String, JobInfo>> it =
        jobHistoryCache.entrySet().iterator();
      String removeJobId = it.next().getKey();
      it.remove();
      LOG.info("Job History file removed form cache "+removeJobId);
    }
    return jobInfo;
  }
}
 
Example #2
Source File: PostExPerformanceDiagnoser.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/**
 * @param jobConfFile - URL pointing to job configuration (job_conf.xml) file
 * @param jobHistoryLogFile - URL pointing to job history log file  
 * @param testsConfFile - file path for test configuration file (optional). 
 * If not specified default path is:$HADOOP_HOME/contrib/vaidya/pxpd_tests_config.xml
 * @param reportFile - file path for storing report (optional)
 */
public PostExPerformanceDiagnoser (String jobConfFile, String jobHistoryFile, InputStream testsConfFileIs,
              String reportFile) throws Exception {
  
  this._jobHistoryFile = jobHistoryFile;
  this._testsConfFileIs = testsConfFileIs;
  this._reportFile = reportFile;
  this._jobConfFile = jobConfFile;
  
  /*
   * Read the job information necessary for post performance analysis
   */
  JobConf jobConf = new JobConf();
  JobInfo jobInfo = new JobInfo("");
  readJobInformation(jobConf, jobInfo);
  this._jobExecutionStatistics = new JobStatistics(jobConf, jobInfo);
}
 
Example #3
Source File: PostExPerformanceDiagnoser.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/**
 * read and populate job statistics information.
 */
private void readJobInformation(JobConf jobConf, JobInfo jobInfo) throws Exception {

  /*
   * Convert the input strings to URL
   */
  URL jobConfFileUrl = new URL(this._jobConfFile);
  URL jobHistoryFileUrl = new URL (this._jobHistoryFile);
  
  /*
   * Read the Job Configuration from the jobConfFile url
   */  
  jobConf.addResource(jobConfFileUrl);
  
  /* 
   * Read JobHistoryFile and build job counters to evaluate diagnostic rules
   */
  if (jobHistoryFileUrl.getProtocol().equals("hdfs")) {
    DefaultJobHistoryParser.parseJobTasks (jobHistoryFileUrl.getPath(), jobInfo, FileSystem.get(jobConf));
  } else if (jobHistoryFileUrl.getProtocol().equals("file")) {
    DefaultJobHistoryParser.parseJobTasks (jobHistoryFileUrl.getPath(), jobInfo, FileSystem.getLocal(jobConf));
  } else {
    throw new Exception("Malformed URL. Protocol: "+jobHistoryFileUrl.getProtocol());
  }
}
 
Example #4
Source File: PostExPerformanceDiagnoser.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
/**
 * @param jobConfFile - URL pointing to job configuration (job_conf.xml) file
 * @param jobHistoryLogFile - URL pointing to job history log file  
 * @param testsConfFile - file path for test configuration file (optional). 
 * If not specified default path is:$HADOOP_HOME/contrib/vaidya/pxpd_tests_config.xml
 * @param reportFile - file path for storing report (optional)
 */
public PostExPerformanceDiagnoser (String jobConfFile, String jobHistoryFile, InputStream testsConfFileIs,
              String reportFile) throws Exception {
  
  this._jobHistoryFile = jobHistoryFile;
  this._testsConfFileIs = testsConfFileIs;
  this._reportFile = reportFile;
  this._jobConfFile = jobConfFile;
  
  /*
   * Read the job information necessary for post performance analysis
   */
  JobConf jobConf = new JobConf();
  JobInfo jobInfo = new JobInfo("");
  readJobInformation(jobConf, jobInfo);
  this._jobExecutionStatistics = new JobStatistics(jobConf, jobInfo);
}
 
Example #5
Source File: PostExPerformanceDiagnoser.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
/**
 * read and populate job statistics information.
 */
private void readJobInformation(JobConf jobConf, JobInfo jobInfo) throws Exception {

  /*
   * Convert the input strings to URL
   */
  URL jobConfFileUrl = new URL(this._jobConfFile);
  URL jobHistoryFileUrl = new URL (this._jobHistoryFile);
  
  /*
   * Read the Job Configuration from the jobConfFile url
   */  
  jobConf.addResource(jobConfFileUrl);
  
  /* 
   * Read JobHistoryFile and build job counters to evaluate diagnostic rules
   */
  if (jobHistoryFileUrl.getProtocol().equals("hdfs")) {
    DefaultJobHistoryParser.parseJobTasks (jobHistoryFileUrl.getPath(), jobInfo, FileSystem.get(jobConf));
  } else if (jobHistoryFileUrl.getProtocol().equals("file")) {
    DefaultJobHistoryParser.parseJobTasks (jobHistoryFileUrl.getPath(), jobInfo, FileSystem.getLocal(jobConf));
  } else {
    throw new Exception("Malformed URL. Protocol: "+jobHistoryFileUrl.getProtocol());
  }
}
 
Example #6
Source File: HadoopJobHistoryLoader.java    From spork with Apache License 2.0 5 votes vote down vote up
public static void parseJobHistory(Configuration jobConf, JobInfo jobInfo, MRJobInfo value) {
    value.job.clear();
    populateJob(jobInfo.getValues(), value.job);
    value.mapTask.clear();
    value.reduceTask.clear();
    populateMapReduceTaskLists(value, jobInfo.getAllTasks());
}
 
Example #7
Source File: TestJobHistoryVersion.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/**
 * Tests the JobHistory parser with different versions of job history files
 */
public void testJobHistoryVersion() throws IOException {
  // If new job history version comes up, the modified parser may fail for
  // the history file created by writeHistoryFile().
  for (long version = 0; version <= JobHistory.VERSION; version++) {
    JobConf conf = new JobConf();
    FileSystem fs = FileSystem.getLocal(conf);
  
    // cleanup
    fs.delete(TEST_DIR, true);
  
    Path historyPath = new Path(TEST_DIR + "/_logs/history/" +
                                FILENAME + version);
  
    fs.delete(historyPath, false);
  
    FSDataOutputStream out = fs.create(historyPath);
    writeHistoryFile(out, version);
    out.close();
  
    JobInfo job = new JobHistory.JobInfo(JOB); 
    DefaultJobHistoryParser.parseJobTasks(historyPath.toString(), job, fs);
  
    assertTrue("Failed to parse jobhistory files of version " + version,
               job.getAllTasks().size() > 0);
  
    // cleanup
    fs.delete(TEST_DIR, true);
  }
}
 
Example #8
Source File: JobStatistics.java    From RDFS with Apache License 2.0 5 votes vote down vote up
public JobStatistics (JobConf jobConf, JobInfo jobInfo) throws ParseException {
  this._jobConf = jobConf;
  this._jobInfo = jobInfo;
  this._job = new Hashtable<Enum, String>();
  populate_Job(this._job, this._jobInfo.getValues());  
  populate_MapReduceTaskLists(this._mapTaskList, this._reduceTaskList, this._jobInfo.getAllTasks());

  // Add the Job Type: MAP_REDUCE, MAP_ONLY
  if (getLongValue(JobKeys.TOTAL_REDUCES) == 0) {
    this._job.put(JobKeys.JOBTYPE,"MAP_ONLY");
  } else {
    this._job.put(JobKeys.JOBTYPE,"MAP_REDUCE");
  }
}
 
Example #9
Source File: TestJobHistoryVersion.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
/**
 * Tests the JobHistory parser with different versions of job history files
 */
public void testJobHistoryVersion() throws IOException {
  // If new job history version comes up, the modified parser may fail for
  // the history file created by writeHistoryFile().
  for (long version = 0; version <= JobHistory.VERSION; version++) {
    JobConf conf = new JobConf();
    FileSystem fs = FileSystem.getLocal(conf);
  
    // cleanup
    fs.delete(TEST_DIR, true);
  
    Path historyPath = new Path(TEST_DIR + "/_logs/history/" +
                                FILENAME + version);
  
    fs.delete(historyPath, false);
  
    FSDataOutputStream out = fs.create(historyPath);
    writeHistoryFile(out, version);
    out.close();
  
    JobInfo job = new JobHistory.JobInfo(JOB); 
    DefaultJobHistoryParser.parseJobTasks(historyPath.toString(), job, fs);
  
    assertTrue("Failed to parse jobhistory files of version " + version,
               job.getAllTasks().size() > 0);
  
    // cleanup
    fs.delete(TEST_DIR, true);
  }
}
 
Example #10
Source File: JobStatistics.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
public JobStatistics (JobConf jobConf, JobInfo jobInfo) throws ParseException {
  this._jobConf = jobConf;
  this._jobInfo = jobInfo;
  this._job = new Hashtable<Enum, String>();
  populate_Job(this._job, this._jobInfo.getValues());
  populate_MapReduceTaskLists(this._mapTaskList, this._reduceTaskList, this._jobInfo.getAllTasks());
}
 
Example #11
Source File: HadoopJobHistoryLoader.java    From spork with Apache License 2.0 4 votes vote down vote up
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
    if (location != null) {
        LOG.info("load: " + location);  
        Path full = new Path(location);  
        String[] jobDetails = 
            JobInfo.decodeJobHistoryFileName(full.getName()).split("_");
        String jobId = jobDetails[2] + "_" + jobDetails[3] + "_"
                + jobDetails[4];
        JobHistory.JobInfo job = new JobHistory.JobInfo(jobId); 
 
        value = new MRJobInfo();
                                    
        FileSystem fs = full.getFileSystem(conf);
        FileStatus fstat = fs.getFileStatus(full);
        
        LOG.info("file size: " + fstat.getLen());
        DefaultJobHistoryParser.parseJobTasks(location, job,
                full.getFileSystem(conf)); 
        LOG.info("job history parsed sucessfully");
        HadoopJobHistoryLoader.parseJobHistory(conf, job, value);
        LOG.info("get parsed job history");
        
        // parse Hadoop job xml file
        Path parent = full.getParent();
        String jobXml = jobDetails[0] + "_" + jobDetails[1] + "_" + jobDetails[2] + "_conf.xml";
        Path p = new Path(parent, jobXml);  
     
        FSDataInputStream fileIn = fs.open(p);
        Map<String, String> val = HadoopJobHistoryLoader
                .parseJobXML(fileIn);
        for (String key : val.keySet()) {
            value.job.put(key, val.get(key));
        }
        
        location = null;
        return true;
    }          
    value = null;
    return false;
}