org.apache.hadoop.mapred.JobID Java Examples
The following examples show how to use
org.apache.hadoop.mapred.JobID.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DataJoinJob.java From hadoop-gpu with Apache License 2.0 | 6 votes |
/** * Submit/run a map/reduce job. * * @param job * @return true for success * @throws IOException */ public static boolean runJob(JobConf job) throws IOException { JobClient jc = new JobClient(job); boolean sucess = true; RunningJob running = null; try { running = jc.submitJob(job); JobID jobId = running.getID(); System.out.println("Job " + jobId + " is submitted"); while (!running.isComplete()) { System.out.println("Job " + jobId + " is still running."); try { Thread.sleep(60000); } catch (InterruptedException e) { } running = jc.getJob(jobId); } sucess = running.isSuccessful(); } finally { if (!sucess && (running != null)) { running.killJob(); } jc.close(); } return sucess; }
Example #2
Source File: DataJoinJob.java From big-c with Apache License 2.0 | 6 votes |
/** * Submit/run a map/reduce job. * * @param job * @return true for success * @throws IOException */ public static boolean runJob(JobConf job) throws IOException { JobClient jc = new JobClient(job); boolean sucess = true; RunningJob running = null; try { running = jc.submitJob(job); JobID jobId = running.getID(); System.out.println("Job " + jobId + " is submitted"); while (!running.isComplete()) { System.out.println("Job " + jobId + " is still running."); try { Thread.sleep(60000); } catch (InterruptedException e) { } running = jc.getJob(jobId); } sucess = running.isSuccessful(); } finally { if (!sucess && (running != null)) { running.killJob(); } jc.close(); } return sucess; }
Example #3
Source File: BoaOutputCommitter.java From compiler with Apache License 2.0 | 6 votes |
@Override public void abortJob(final JobContext context, final JobStatus.State runState) throws java.io.IOException { super.abortJob(context, runState); final JobClient jobClient = new JobClient(new JobConf(context.getConfiguration())); final RunningJob job = jobClient.getJob((org.apache.hadoop.mapred.JobID) JobID.forName(context.getConfiguration().get("mapred.job.id"))); String diag = ""; for (final TaskCompletionEvent event : job.getTaskCompletionEvents(0)) switch (event.getTaskStatus()) { case SUCCEEDED: break; default: diag += "Diagnostics for: " + event.getTaskTrackerHttp() + "\n"; for (final String s : job.getTaskDiagnostics(event.getTaskAttemptId())) diag += s + "\n"; diag += "\n"; break; } updateStatus(diag, context.getConfiguration().getInt("boa.hadoop.jobid", 0)); }
Example #4
Source File: TestJobControl.java From big-c with Apache License 2.0 | 6 votes |
@SuppressWarnings("deprecation") @Test(timeout = 30000) public void testJobState() throws Exception { Job job_1 = getCopyJob(); JobControl jc = new JobControl("Test"); jc.addJob(job_1); Assert.assertEquals(Job.WAITING, job_1.getState()); job_1.setState(Job.SUCCESS); Assert.assertEquals(Job.WAITING, job_1.getState()); org.apache.hadoop.mapreduce.Job mockjob = mock(org.apache.hadoop.mapreduce.Job.class); org.apache.hadoop.mapreduce.JobID jid = new org.apache.hadoop.mapreduce.JobID("test", 0); when(mockjob.getJobID()).thenReturn(jid); job_1.setJob(mockjob); Assert.assertEquals("job_test_0000", job_1.getMapredJobID()); job_1.setMapredJobID("job_test_0001"); Assert.assertEquals("job_test_0000", job_1.getMapredJobID()); jc.stop(); }
Example #5
Source File: DataJoinJob.java From hadoop with Apache License 2.0 | 6 votes |
/** * Submit/run a map/reduce job. * * @param job * @return true for success * @throws IOException */ public static boolean runJob(JobConf job) throws IOException { JobClient jc = new JobClient(job); boolean sucess = true; RunningJob running = null; try { running = jc.submitJob(job); JobID jobId = running.getID(); System.out.println("Job " + jobId + " is submitted"); while (!running.isComplete()) { System.out.println("Job " + jobId + " is still running."); try { Thread.sleep(60000); } catch (InterruptedException e) { } running = jc.getJob(jobId); } sucess = running.isSuccessful(); } finally { if (!sucess && (running != null)) { running.killJob(); } jc.close(); } return sucess; }
Example #6
Source File: MapReduceLauncher.java From spork with Apache License 2.0 | 6 votes |
@Override public void killJob(String jobID, Configuration conf) throws BackendException { try { if (conf != null) { JobConf jobConf = new JobConf(conf); JobClient jc = new JobClient(jobConf); JobID id = JobID.forName(jobID); RunningJob job = jc.getJob(id); if (job == null) System.out.println("Job with id " + jobID + " is not active"); else { job.killJob(); log.info("Kill " + id + " submitted."); } } } catch (IOException e) { throw new BackendException(e); } }
Example #7
Source File: TestJobControl.java From hadoop with Apache License 2.0 | 6 votes |
@SuppressWarnings("deprecation") @Test(timeout = 30000) public void testJobState() throws Exception { Job job_1 = getCopyJob(); JobControl jc = new JobControl("Test"); jc.addJob(job_1); Assert.assertEquals(Job.WAITING, job_1.getState()); job_1.setState(Job.SUCCESS); Assert.assertEquals(Job.WAITING, job_1.getState()); org.apache.hadoop.mapreduce.Job mockjob = mock(org.apache.hadoop.mapreduce.Job.class); org.apache.hadoop.mapreduce.JobID jid = new org.apache.hadoop.mapreduce.JobID("test", 0); when(mockjob.getJobID()).thenReturn(jid); job_1.setJob(mockjob); Assert.assertEquals("job_test_0000", job_1.getMapredJobID()); job_1.setMapredJobID("job_test_0001"); Assert.assertEquals("job_test_0000", job_1.getMapredJobID()); jc.stop(); }
Example #8
Source File: TestNoJobSetupCleanup.java From RDFS with Apache License 2.0 | 6 votes |
private Job submitAndValidateJob(JobConf conf, int numMaps, int numReds) throws IOException, InterruptedException, ClassNotFoundException { conf.setJobSetupCleanupNeeded(false); Job job = MapReduceTestUtil.createJob(conf, inDir, outDir, numMaps, numReds); job.setOutputFormatClass(MyOutputFormat.class); job.waitForCompletion(true); assertTrue(job.isSuccessful()); JobID jobid = (org.apache.hadoop.mapred.JobID)job.getID(); JobClient jc = new JobClient(conf); assertTrue(jc.getSetupTaskReports(jobid).length == 0); assertTrue(jc.getCleanupTaskReports(jobid).length == 0); assertTrue(jc.getMapTaskReports(jobid).length == numMaps); assertTrue(jc.getReduceTaskReports(jobid).length == numReds); FileSystem fs = FileSystem.get(conf); assertTrue("Job output directory doesn't exit!", fs.exists(outDir)); FileStatus[] list = fs.listStatus(outDir, new OutputFilter()); int numPartFiles = numReds == 0 ? numMaps : numReds; assertTrue("Number of part-files is " + list.length + " and not " + numPartFiles, list.length == numPartFiles); return job; }
Example #9
Source File: DataJoinJob.java From RDFS with Apache License 2.0 | 6 votes |
/** * Submit/run a map/reduce job. * * @param job * @return true for success * @throws IOException */ public static boolean runJob(JobConf job) throws IOException { JobClient jc = new JobClient(job); boolean sucess = true; RunningJob running = null; try { running = jc.submitJob(job); JobID jobId = running.getID(); System.out.println("Job " + jobId + " is submitted"); while (!running.isComplete()) { System.out.println("Job " + jobId + " is still running."); try { Thread.sleep(60000); } catch (InterruptedException e) { } running = jc.getJob(jobId); } sucess = running.isSuccessful(); } finally { if (!sucess && (running != null)) { running.killJob(); } jc.close(); } return sucess; }
Example #10
Source File: TestMRJobStats.java From spork with Apache License 2.0 | 6 votes |
@Test public void testMedianMapReduceTime() throws Exception { JobClient jobClient = Mockito.mock(JobClient.class); // mock methods to return the predefined map and reduce task reports Mockito.when(jobClient.getMapTaskReports(jobID)).thenReturn(mapTaskReports); Mockito.when(jobClient.getReduceTaskReports(jobID)).thenReturn(reduceTaskReports); PigStats.JobGraph jobGraph = new PigStats.JobGraph(); MRJobStats jobStats = createJobStats("JobStatsTest", jobGraph); getJobStatsMethod("setId", JobID.class).invoke(jobStats, jobID); jobStats.setSuccessful(true); getJobStatsMethod("addMapReduceStatistics", Iterator.class, Iterator.class) .invoke(jobStats, Arrays.asList(mapTaskReports).iterator(), Arrays.asList(reduceTaskReports).iterator()); String msg = (String)getJobStatsMethod("getDisplayString") .invoke(jobStats); System.out.println(JobStats.SUCCESS_HEADER); System.out.println(msg); assertTrue(msg.startsWith(ASSERT_STRING)); }
Example #11
Source File: HadoopV2TaskContext.java From ignite with Apache License 2.0 | 5 votes |
/** * @param taskInfo Task info. * @param job Job. * @param jobId Job ID. * @param locNodeId Local node ID. * @param jobConfDataInput DataInput for read JobConf. */ public HadoopV2TaskContext(HadoopTaskInfo taskInfo, HadoopJobEx job, HadoopJobId jobId, @Nullable UUID locNodeId, DataInput jobConfDataInput) throws IgniteCheckedException { super(taskInfo, job); this.locNodeId = locNodeId; // Before create JobConf instance we should set new context class loader. ClassLoader oldLdr = HadoopCommonUtils.setContextClassLoader(getClass().getClassLoader()); try { JobConf jobConf = new JobConf(); try { jobConf.readFields(jobConfDataInput); } catch (IOException e) { throw new IgniteCheckedException(e); } // For map-reduce jobs prefer local writes. jobConf.setBooleanIfUnset(PARAM_IGFS_PREFER_LOCAL_WRITES, true); initializePartiallyRawComparator(jobConf); jobCtx = new JobContextImpl(jobConf, new JobID(jobId.globalId().toString(), jobId.localId())); useNewMapper = jobConf.getUseNewMapper(); useNewReducer = jobConf.getUseNewReducer(); useNewCombiner = jobConf.getCombinerClass() == null; } finally { HadoopCommonUtils.restoreContextClassLoader(oldLdr); } }
Example #12
Source File: HadoopOutputFormatBase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public void finalizeGlobal(int parallelism) throws IOException { try { JobContext jobContext = new JobContextImpl(this.jobConf, new JobID()); OutputCommitter outputCommitter = this.jobConf.getOutputCommitter(); // finalize HDFS output format outputCommitter.commitJob(jobContext); } catch (Exception e) { throw new RuntimeException(e); } }
Example #13
Source File: HadoopOutputFormatBase.java From flink with Apache License 2.0 | 5 votes |
/** * create the temporary output file for hadoop RecordWriter. * @param taskNumber The number of the parallel instance. * @param numTasks The number of parallel tasks. * @throws java.io.IOException */ @Override public void open(int taskNumber, int numTasks) throws IOException { // enforce sequential open() calls synchronized (OPEN_MUTEX) { if (Integer.toString(taskNumber + 1).length() > 6) { throw new IOException("Task id too large."); } TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_" + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s", " ").replace(" ", "0") + Integer.toString(taskNumber + 1) + "_0"); this.jobConf.set("mapred.task.id", taskAttemptID.toString()); this.jobConf.setInt("mapred.task.partition", taskNumber + 1); // for hadoop 2.2 this.jobConf.set("mapreduce.task.attempt.id", taskAttemptID.toString()); this.jobConf.setInt("mapreduce.task.partition", taskNumber + 1); this.context = new TaskAttemptContextImpl(this.jobConf, taskAttemptID); this.outputCommitter = this.jobConf.getOutputCommitter(); JobContext jobContext = new JobContextImpl(this.jobConf, new JobID()); this.outputCommitter.setupJob(jobContext); this.recordWriter = this.mapredOutputFormat.getRecordWriter(null, this.jobConf, Integer.toString(taskNumber + 1), new HadoopDummyProgressable()); } }
Example #14
Source File: HadoopSource.java From twister2 with Apache License 2.0 | 5 votes |
@Override public void prepare(TSetContext ctx) { this.context = ctx; Configuration hadoopConf = this.wrappedConfiguration.getConfiguration(); jconf = new JobConf(hadoopConf); try { format = inputClazz.newInstance(); JobContext jobContext = new JobContextImpl(hadoopConf, new JobID(context.getId(), context.getIndex())); List<InputSplit> splits = format.getSplits(jobContext); for (int i = 0; i < splits.size(); i++) { if (i % context.getParallelism() == context.getIndex()) { assignedSplits.add(splits.get(i)); } } if (assignedSplits.size() > 0) { TaskID taskID = new TaskID(context.getId(), context.getIndex(), TaskType.MAP, context.getIndex()); TaskAttemptID taskAttemptID = new TaskAttemptID(taskID, context.getIndex()); TaskAttemptContextImpl taskAttemptContext = new TaskAttemptContextImpl(jconf, taskAttemptID); currentReader = format.createRecordReader(assignedSplits.get(consumingSplit), taskAttemptContext); currentReader.initialize(assignedSplits.get(consumingSplit), taskAttemptContext); } } catch (InstantiationException | IllegalAccessException | InterruptedException | IOException e) { throw new RuntimeException("Failed to initialize hadoop input", e); } }
Example #15
Source File: HadoopSourceWithMap.java From twister2 with Apache License 2.0 | 5 votes |
@Override public void prepare(TSetContext ctx) { this.context = ctx; Configuration hadoopConf = this.wrappedConfiguration.getConfiguration(); jconf = new JobConf(hadoopConf); try { format = inputClazz.newInstance(); JobContext jobContext = new JobContextImpl(hadoopConf, new JobID(context.getId(), context.getIndex())); List<InputSplit> splits = format.getSplits(jobContext); for (int i = 0; i < splits.size(); i++) { if (i % context.getParallelism() == context.getIndex()) { assignedSplits.add(splits.get(i)); } } if (assignedSplits.size() > 0) { TaskID taskID = new TaskID(context.getId(), context.getIndex(), TaskType.MAP, context.getIndex()); TaskAttemptID taskAttemptID = new TaskAttemptID(taskID, context.getIndex()); TaskAttemptContextImpl taskAttemptContext = new TaskAttemptContextImpl(jconf, taskAttemptID); currentReader = format.createRecordReader(assignedSplits.get(consumingSplit), taskAttemptContext); currentReader.initialize(assignedSplits.get(consumingSplit), taskAttemptContext); } } catch (InstantiationException | IllegalAccessException | InterruptedException | IOException e) { throw new RuntimeException("Failed to initialize hadoop input", e); } }
Example #16
Source File: TestJobControl.java From hadoop with Apache License 2.0 | 5 votes |
@Test (timeout = 30000) public void testGetAssignedJobId() throws Exception { JobConf jc = new JobConf(); Job j = new Job(jc); //Just make sure no exception is thrown assertNull(j.getAssignedJobID()); org.apache.hadoop.mapreduce.Job mockjob = mock(org.apache.hadoop.mapreduce.Job.class); org.apache.hadoop.mapreduce.JobID jid = new org.apache.hadoop.mapreduce.JobID("test",0); when(mockjob.getJobID()).thenReturn(jid); j.setJob(mockjob); JobID expected = new JobID("test",0); assertEquals(expected, j.getAssignedJobID()); verify(mockjob).getJobID(); }
Example #17
Source File: Job.java From hadoop with Apache License 2.0 | 5 votes |
/** * @return the mapred ID of this job as assigned by the mapred framework. */ public JobID getAssignedJobID() { org.apache.hadoop.mapreduce.JobID temp = super.getMapredJobId(); if (temp == null) { return null; } return JobID.downgrade(temp); }
Example #18
Source File: HistoryLogUtils.java From spydra with Apache License 2.0 | 5 votes |
public static Optional<String> findHistoryFilePath( Iterator<LocatedFileStatus> listing, ApplicationId applicationId) { JobID jobId = new JobID( String.valueOf(applicationId.getClusterTimestamp()), applicationId.getId()); List<LocatedFileStatus> jhistFiles = new ArrayList<>(); // maybe this could work more nicely with some recursive glob and a filter try { jhistFiles = StreamSupport .stream(Spliterators.spliteratorUnknownSize(listing, Spliterator.NONNULL), false) .filter(fstatus -> fstatus.getPath().toString() .matches(".*" + jobId.toString() + ".*.jhist")) .collect(Collectors.toList()); } catch (RemoteIteratorAdaptor.WrappedRemoteIteratorException wrie) { // We can't really do overly much at this point, as this is an error from the // underlying hadoop filesystem implementation. But we want to at least log this // separately from other conditions. logger.error("Retrieving remote listing failed", wrie); } if (jhistFiles.size() < 1) { logger.error("Could not locate a history file for parameters"); return Optional.empty(); } else if (jhistFiles.size() > 1) { logger.error("Found two or more matching files, will dump first"); } return jhistFiles.stream() .findFirst() .map(x -> x.getPath().toString()); }
Example #19
Source File: HadoopOutputFormatBase.java From flink with Apache License 2.0 | 5 votes |
@Override public void finalizeGlobal(int parallelism) throws IOException { try { JobContext jobContext = new JobContextImpl(this.jobConf, new JobID()); OutputCommitter outputCommitter = this.jobConf.getOutputCommitter(); // finalize HDFS output format outputCommitter.commitJob(jobContext); } catch (Exception e) { throw new RuntimeException(e); } }
Example #20
Source File: HadoopUtils.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Nonnull public static String getJobIdFromTaskId(@Nonnull String taskidStr) { if (!taskidStr.startsWith("task_")) {// workaround for Tez taskidStr = taskidStr.replace("task", "task_"); taskidStr = taskidStr.substring(0, taskidStr.lastIndexOf('_')); } TaskID taskId = TaskID.forName(taskidStr); JobID jobId = taskId.getJobID(); return jobId.toString(); }
Example #21
Source File: MapreduceSubmissionEngine.java From sqoop-on-spark with Apache License 2.0 | 5 votes |
/** * {@inheritDoc} */ @Override public void stop(String externalJobId) { try { RunningJob runningJob = jobClient.getJob(JobID.forName(externalJobId)); if(runningJob == null) { return; } runningJob.killJob(); } catch (IOException e) { throw new SqoopException(MapreduceSubmissionError.MAPREDUCE_0003, e); } }
Example #22
Source File: MapreduceSubmissionEngine.java From sqoop-on-spark with Apache License 2.0 | 5 votes |
/** * {@inheritDoc} */ @Override public void update(MSubmission submission) { double progress = -1; Counters counters = null; String externalJobId = submission.getExternalJobId(); try { RunningJob runningJob = jobClient.getJob(JobID.forName(externalJobId)); SubmissionStatus newStatus = status(runningJob); SubmissionError error = error(runningJob); if (newStatus.isRunning()) { progress = progress(runningJob); } else { counters = counters(runningJob); } // these properties change as the job runs, rest of the submission attributes // do not change as job runs submission.setStatus(newStatus); submission.setError(error); submission.setProgress(progress); submission.setCounters(counters); submission.setLastUpdateDate(new Date()); } catch (IOException e) { throw new SqoopException(MapreduceSubmissionError.MAPREDUCE_0003, e); } }
Example #23
Source File: HadoopJob.java From hadoop-gpu with Apache License 2.0 | 5 votes |
/** * Constructor for a Hadoop job representation * * @param location * @param id * @param running * @param status */ public HadoopJob(HadoopServer location, JobID id, RunningJob running, JobStatus status) { this.location = location; this.jobId = id; this.running = running; loadJobFile(); update(status); }
Example #24
Source File: HadoopJob.java From RDFS with Apache License 2.0 | 5 votes |
/** * Constructor for a Hadoop job representation * * @param location * @param id * @param running * @param status */ public HadoopJob(HadoopServer location, JobID id, RunningJob running, JobStatus status) { this.location = location; this.jobId = id; this.running = running; loadJobFile(); update(status); }
Example #25
Source File: PigJobServerImpl.java From oink with Apache License 2.0 | 5 votes |
@Override public boolean cancelRequest(String requestId) throws IOException { PigRequestStats stats = this.getRequestStats(requestId); if (stats.getStatus().equals(Status.SUBMITTED.toString())) { List<String> jobs= stats.getJobs(); for (String job : jobs) { job= job.substring(JT_UI.length()); JobConf jobConf = new JobConf(); jobConf.set("fs.default.name", PropertyLoader.getInstance().getProperty("fs.default.name")); jobConf.set("mapred.job.tracker", PropertyLoader.getInstance().getProperty("jobtracker")); try { JobClient jobClient = new JobClient(jobConf); RunningJob rJob = jobClient.getJob(JobID.forName(job)); if (! rJob.isComplete()) { rJob.killJob(); } } catch (Exception e) { throw new IOException ("Unable to kill job " + job); } } PigRequestStats requestStats= new PigRequestStats(0, 0, null, jobs.size()); requestStats.setJobs(jobs); requestStats.setStatus(Status.KILLED.toString()); Path statsPath= new Path(PropertyLoader.getInstance().getProperty(Constants.REQUEST_PATH) + requestId + "/stats"); PigUtils.writeStatsFile(statsPath, requestStats); return true; } else { return false; } }
Example #26
Source File: HadoopOutputFormatBase.java From flink with Apache License 2.0 | 5 votes |
/** * create the temporary output file for hadoop RecordWriter. * @param taskNumber The number of the parallel instance. * @param numTasks The number of parallel tasks. * @throws java.io.IOException */ @Override public void open(int taskNumber, int numTasks) throws IOException { // enforce sequential open() calls synchronized (OPEN_MUTEX) { if (Integer.toString(taskNumber + 1).length() > 6) { throw new IOException("Task id too large."); } TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_" + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s", " ").replace(" ", "0") + Integer.toString(taskNumber + 1) + "_0"); this.jobConf.set("mapred.task.id", taskAttemptID.toString()); this.jobConf.setInt("mapred.task.partition", taskNumber + 1); // for hadoop 2.2 this.jobConf.set("mapreduce.task.attempt.id", taskAttemptID.toString()); this.jobConf.setInt("mapreduce.task.partition", taskNumber + 1); this.context = new TaskAttemptContextImpl(this.jobConf, taskAttemptID); this.outputCommitter = this.jobConf.getOutputCommitter(); JobContext jobContext = new JobContextImpl(this.jobConf, new JobID()); this.outputCommitter.setupJob(jobContext); this.recordWriter = this.mapredOutputFormat.getRecordWriter(null, this.jobConf, Integer.toString(taskNumber + 1), new HadoopDummyProgressable()); } }
Example #27
Source File: HadoopOutputFormatBase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * create the temporary output file for hadoop RecordWriter. * @param taskNumber The number of the parallel instance. * @param numTasks The number of parallel tasks. * @throws java.io.IOException */ @Override public void open(int taskNumber, int numTasks) throws IOException { // enforce sequential open() calls synchronized (OPEN_MUTEX) { if (Integer.toString(taskNumber + 1).length() > 6) { throw new IOException("Task id too large."); } TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_" + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s", " ").replace(" ", "0") + Integer.toString(taskNumber + 1) + "_0"); this.jobConf.set("mapred.task.id", taskAttemptID.toString()); this.jobConf.setInt("mapred.task.partition", taskNumber + 1); // for hadoop 2.2 this.jobConf.set("mapreduce.task.attempt.id", taskAttemptID.toString()); this.jobConf.setInt("mapreduce.task.partition", taskNumber + 1); this.context = new TaskAttemptContextImpl(this.jobConf, taskAttemptID); this.outputCommitter = this.jobConf.getOutputCommitter(); JobContext jobContext = new JobContextImpl(this.jobConf, new JobID()); this.outputCommitter.setupJob(jobContext); this.recordWriter = this.mapredOutputFormat.getRecordWriter(null, this.jobConf, Integer.toString(taskNumber + 1), new HadoopDummyProgressable()); } }
Example #28
Source File: SimplePigStats.java From spork with Apache License 2.0 | 5 votes |
MRJobStats addMRJobStats(Job job) { MapReduceOper mro = jobMroMap.get(job); if (mro == null) { LOG.warn("unable to get MR oper for job: " + job.toString()); return null; } MRJobStats js = mroJobMap.get(mro); JobID jobId = job.getAssignedJobID(); js.setId(jobId); js.setAlias(mro); js.setConf(job.getJobConf()); return js; }
Example #29
Source File: SimplePigStats.java From spork with Apache License 2.0 | 5 votes |
MRJobStats addMRJobStatsForNative(NativeMapReduceOper mr) { MRJobStats js = mroJobMap.get(mr); js.setId(new JobID(mr.getJobId(), NativeMapReduceOper.getJobNumber())); js.setAlias(mr); return js; }
Example #30
Source File: TestMRJobStats.java From spork with Apache License 2.0 | 5 votes |
@Test public void testOneTaskReport() throws Exception { // setting up one map task report TaskReport[] mapTaskReports = new TaskReport[1]; mapTaskReports[0] = Mockito.mock(TaskReport.class); Mockito.when(mapTaskReports[0].getStartTime()).thenReturn(300L * ONE_THOUSAND); Mockito.when(mapTaskReports[0].getFinishTime()).thenReturn(400L * ONE_THOUSAND); // setting up one reduce task report TaskReport[] reduceTaskReports = new TaskReport[1]; reduceTaskReports[0] = Mockito.mock(TaskReport.class); Mockito.when(reduceTaskReports[0].getStartTime()).thenReturn(500L * ONE_THOUSAND); Mockito.when(reduceTaskReports[0].getFinishTime()).thenReturn(700L * ONE_THOUSAND); PigStats.JobGraph jobGraph = new PigStats.JobGraph(); MRJobStats jobStats = createJobStats("JobStatsTest", jobGraph); getJobStatsMethod("setId", JobID.class).invoke(jobStats, jobID); jobStats.setSuccessful(true); getJobStatsMethod("addMapReduceStatistics", Iterator.class, Iterator.class) .invoke(jobStats, Arrays.asList(mapTaskReports).iterator(), Arrays.asList(reduceTaskReports).iterator()); String msg = (String)getJobStatsMethod("getDisplayString") .invoke(jobStats); System.out.println(JobStats.SUCCESS_HEADER); System.out.println(msg); StringBuilder sb = new StringBuilder(); sb.append(jobID.toString()).append("\t"); sb.append(mapTaskReports.length).append("\t"); sb.append(reduceTaskReports.length).append("\t"); sb.append("100\t100\t100\t100\t200\t200\t200\t200"); System.out.println("assert msg: " + sb.toString()); assertTrue(msg.startsWith(sb.toString())); }