Java Code Examples for org.apache.hadoop.mapred.JobHistory.JobInfo

The following examples show how to use org.apache.hadoop.mapred.JobHistory.JobInfo. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: RDFS   Source File: JSPUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static JobInfo getJobInfo(HttpServletRequest request, FileSystem fs)
    throws IOException {
  String jobid = request.getParameter("jobid");
  String logFile = request.getParameter("logFile");
  synchronized(jobHistoryCache) {
    JobInfo jobInfo = jobHistoryCache.remove(jobid);
    if (jobInfo == null) {
      jobInfo = new JobHistory.JobInfo(jobid);
      LOG.info("Loading Job History file "+jobid + ".   Cache size is " +
          jobHistoryCache.size());
      DefaultJobHistoryParser.parseJobTasks( logFile, jobInfo, fs) ;
    }
    jobHistoryCache.put(jobid, jobInfo);
    if (jobHistoryCache.size() > CACHE_SIZE) {
      Iterator<Map.Entry<String, JobInfo>> it =
        jobHistoryCache.entrySet().iterator();
      String removeJobId = it.next().getKey();
      it.remove();
      LOG.info("Job History file removed form cache "+removeJobId);
    }
    return jobInfo;
  }
}
 
Example 2
Source Project: RDFS   Source File: PostExPerformanceDiagnoser.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @param jobConfFile - URL pointing to job configuration (job_conf.xml) file
 * @param jobHistoryLogFile - URL pointing to job history log file  
 * @param testsConfFile - file path for test configuration file (optional). 
 * If not specified default path is:$HADOOP_HOME/contrib/vaidya/pxpd_tests_config.xml
 * @param reportFile - file path for storing report (optional)
 */
public PostExPerformanceDiagnoser (String jobConfFile, String jobHistoryFile, InputStream testsConfFileIs,
              String reportFile) throws Exception {
  
  this._jobHistoryFile = jobHistoryFile;
  this._testsConfFileIs = testsConfFileIs;
  this._reportFile = reportFile;
  this._jobConfFile = jobConfFile;
  
  /*
   * Read the job information necessary for post performance analysis
   */
  JobConf jobConf = new JobConf();
  JobInfo jobInfo = new JobInfo("");
  readJobInformation(jobConf, jobInfo);
  this._jobExecutionStatistics = new JobStatistics(jobConf, jobInfo);
}
 
Example 3
Source Project: RDFS   Source File: PostExPerformanceDiagnoser.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * read and populate job statistics information.
 */
private void readJobInformation(JobConf jobConf, JobInfo jobInfo) throws Exception {

  /*
   * Convert the input strings to URL
   */
  URL jobConfFileUrl = new URL(this._jobConfFile);
  URL jobHistoryFileUrl = new URL (this._jobHistoryFile);
  
  /*
   * Read the Job Configuration from the jobConfFile url
   */  
  jobConf.addResource(jobConfFileUrl);
  
  /* 
   * Read JobHistoryFile and build job counters to evaluate diagnostic rules
   */
  if (jobHistoryFileUrl.getProtocol().equals("hdfs")) {
    DefaultJobHistoryParser.parseJobTasks (jobHistoryFileUrl.getPath(), jobInfo, FileSystem.get(jobConf));
  } else if (jobHistoryFileUrl.getProtocol().equals("file")) {
    DefaultJobHistoryParser.parseJobTasks (jobHistoryFileUrl.getPath(), jobInfo, FileSystem.getLocal(jobConf));
  } else {
    throw new Exception("Malformed URL. Protocol: "+jobHistoryFileUrl.getProtocol());
  }
}
 
Example 4
Source Project: hadoop-gpu   Source File: PostExPerformanceDiagnoser.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @param jobConfFile - URL pointing to job configuration (job_conf.xml) file
 * @param jobHistoryLogFile - URL pointing to job history log file  
 * @param testsConfFile - file path for test configuration file (optional). 
 * If not specified default path is:$HADOOP_HOME/contrib/vaidya/pxpd_tests_config.xml
 * @param reportFile - file path for storing report (optional)
 */
public PostExPerformanceDiagnoser (String jobConfFile, String jobHistoryFile, InputStream testsConfFileIs,
              String reportFile) throws Exception {
  
  this._jobHistoryFile = jobHistoryFile;
  this._testsConfFileIs = testsConfFileIs;
  this._reportFile = reportFile;
  this._jobConfFile = jobConfFile;
  
  /*
   * Read the job information necessary for post performance analysis
   */
  JobConf jobConf = new JobConf();
  JobInfo jobInfo = new JobInfo("");
  readJobInformation(jobConf, jobInfo);
  this._jobExecutionStatistics = new JobStatistics(jobConf, jobInfo);
}
 
Example 5
Source Project: hadoop-gpu   Source File: PostExPerformanceDiagnoser.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * read and populate job statistics information.
 */
private void readJobInformation(JobConf jobConf, JobInfo jobInfo) throws Exception {

  /*
   * Convert the input strings to URL
   */
  URL jobConfFileUrl = new URL(this._jobConfFile);
  URL jobHistoryFileUrl = new URL (this._jobHistoryFile);
  
  /*
   * Read the Job Configuration from the jobConfFile url
   */  
  jobConf.addResource(jobConfFileUrl);
  
  /* 
   * Read JobHistoryFile and build job counters to evaluate diagnostic rules
   */
  if (jobHistoryFileUrl.getProtocol().equals("hdfs")) {
    DefaultJobHistoryParser.parseJobTasks (jobHistoryFileUrl.getPath(), jobInfo, FileSystem.get(jobConf));
  } else if (jobHistoryFileUrl.getProtocol().equals("file")) {
    DefaultJobHistoryParser.parseJobTasks (jobHistoryFileUrl.getPath(), jobInfo, FileSystem.getLocal(jobConf));
  } else {
    throw new Exception("Malformed URL. Protocol: "+jobHistoryFileUrl.getProtocol());
  }
}
 
Example 6
Source Project: spork   Source File: HadoopJobHistoryLoader.java    License: Apache License 2.0 5 votes vote down vote up
public static void parseJobHistory(Configuration jobConf, JobInfo jobInfo, MRJobInfo value) {
    value.job.clear();
    populateJob(jobInfo.getValues(), value.job);
    value.mapTask.clear();
    value.reduceTask.clear();
    populateMapReduceTaskLists(value, jobInfo.getAllTasks());
}
 
Example 7
Source Project: RDFS   Source File: TestJobHistoryVersion.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests the JobHistory parser with different versions of job history files
 */
public void testJobHistoryVersion() throws IOException {
  // If new job history version comes up, the modified parser may fail for
  // the history file created by writeHistoryFile().
  for (long version = 0; version <= JobHistory.VERSION; version++) {
    JobConf conf = new JobConf();
    FileSystem fs = FileSystem.getLocal(conf);
  
    // cleanup
    fs.delete(TEST_DIR, true);
  
    Path historyPath = new Path(TEST_DIR + "/_logs/history/" +
                                FILENAME + version);
  
    fs.delete(historyPath, false);
  
    FSDataOutputStream out = fs.create(historyPath);
    writeHistoryFile(out, version);
    out.close();
  
    JobInfo job = new JobHistory.JobInfo(JOB); 
    DefaultJobHistoryParser.parseJobTasks(historyPath.toString(), job, fs);
  
    assertTrue("Failed to parse jobhistory files of version " + version,
               job.getAllTasks().size() > 0);
  
    // cleanup
    fs.delete(TEST_DIR, true);
  }
}
 
Example 8
Source Project: RDFS   Source File: JobStatistics.java    License: Apache License 2.0 5 votes vote down vote up
public JobStatistics (JobConf jobConf, JobInfo jobInfo) throws ParseException {
  this._jobConf = jobConf;
  this._jobInfo = jobInfo;
  this._job = new Hashtable<Enum, String>();
  populate_Job(this._job, this._jobInfo.getValues());  
  populate_MapReduceTaskLists(this._mapTaskList, this._reduceTaskList, this._jobInfo.getAllTasks());

  // Add the Job Type: MAP_REDUCE, MAP_ONLY
  if (getLongValue(JobKeys.TOTAL_REDUCES) == 0) {
    this._job.put(JobKeys.JOBTYPE,"MAP_ONLY");
  } else {
    this._job.put(JobKeys.JOBTYPE,"MAP_REDUCE");
  }
}
 
Example 9
Source Project: hadoop-gpu   Source File: TestJobHistoryVersion.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests the JobHistory parser with different versions of job history files
 */
public void testJobHistoryVersion() throws IOException {
  // If new job history version comes up, the modified parser may fail for
  // the history file created by writeHistoryFile().
  for (long version = 0; version <= JobHistory.VERSION; version++) {
    JobConf conf = new JobConf();
    FileSystem fs = FileSystem.getLocal(conf);
  
    // cleanup
    fs.delete(TEST_DIR, true);
  
    Path historyPath = new Path(TEST_DIR + "/_logs/history/" +
                                FILENAME + version);
  
    fs.delete(historyPath, false);
  
    FSDataOutputStream out = fs.create(historyPath);
    writeHistoryFile(out, version);
    out.close();
  
    JobInfo job = new JobHistory.JobInfo(JOB); 
    DefaultJobHistoryParser.parseJobTasks(historyPath.toString(), job, fs);
  
    assertTrue("Failed to parse jobhistory files of version " + version,
               job.getAllTasks().size() > 0);
  
    // cleanup
    fs.delete(TEST_DIR, true);
  }
}
 
Example 10
Source Project: hadoop-gpu   Source File: JobStatistics.java    License: Apache License 2.0 5 votes vote down vote up
public JobStatistics (JobConf jobConf, JobInfo jobInfo) throws ParseException {
  this._jobConf = jobConf;
  this._jobInfo = jobInfo;
  this._job = new Hashtable<Enum, String>();
  populate_Job(this._job, this._jobInfo.getValues());
  populate_MapReduceTaskLists(this._mapTaskList, this._reduceTaskList, this._jobInfo.getAllTasks());
}
 
Example 11
Source Project: spork   Source File: HadoopJobHistoryLoader.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
    if (location != null) {
        LOG.info("load: " + location);  
        Path full = new Path(location);  
        String[] jobDetails = 
            JobInfo.decodeJobHistoryFileName(full.getName()).split("_");
        String jobId = jobDetails[2] + "_" + jobDetails[3] + "_"
                + jobDetails[4];
        JobHistory.JobInfo job = new JobHistory.JobInfo(jobId); 
 
        value = new MRJobInfo();
                                    
        FileSystem fs = full.getFileSystem(conf);
        FileStatus fstat = fs.getFileStatus(full);
        
        LOG.info("file size: " + fstat.getLen());
        DefaultJobHistoryParser.parseJobTasks(location, job,
                full.getFileSystem(conf)); 
        LOG.info("job history parsed sucessfully");
        HadoopJobHistoryLoader.parseJobHistory(conf, job, value);
        LOG.info("get parsed job history");
        
        // parse Hadoop job xml file
        Path parent = full.getParent();
        String jobXml = jobDetails[0] + "_" + jobDetails[1] + "_" + jobDetails[2] + "_conf.xml";
        Path p = new Path(parent, jobXml);  
     
        FSDataInputStream fileIn = fs.open(p);
        Map<String, String> val = HadoopJobHistoryLoader
                .parseJobXML(fileIn);
        for (String key : val.keySet()) {
            value.job.put(key, val.get(key));
        }
        
        location = null;
        return true;
    }          
    value = null;
    return false;
}