Java Code Examples for org.apache.hadoop.mapreduce.TaskID#getId()

The following examples show how to use org.apache.hadoop.mapreduce.TaskID#getId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FileOutputFormat.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * Generate a unique filename, based on the task id, name, and extension
 * @param context the task that is calling this
 * @param name the base filename
 * @param extension the filename extension
 * @return a string like $name-[mrsct]-$id$extension
 */
public synchronized static String getUniqueFile(TaskAttemptContext context,
                                                String name,
                                                String extension) {
  TaskID taskId = context.getTaskAttemptID().getTaskID();
  int partition = taskId.getId();
  StringBuilder result = new StringBuilder();
  result.append(name);
  result.append('-');
  result.append(
      TaskID.getRepresentingCharacter(taskId.getTaskType()));
  result.append('-');
  result.append(NUMBER_FORMAT.format(partition));
  result.append(extension);
  return result.toString();
}
Example 2
Source File: FileOutputFormat.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * Generate a unique filename, based on the task id, name, and extension
 * @param context the task that is calling this
 * @param name the base filename
 * @param extension the filename extension
 * @return a string like $name-[mrsct]-$id$extension
 */
public synchronized static String getUniqueFile(TaskAttemptContext context,
                                                String name,
                                                String extension) {
  TaskID taskId = context.getTaskAttemptID().getTaskID();
  int partition = taskId.getId();
  StringBuilder result = new StringBuilder();
  result.append(name);
  result.append('-');
  result.append(
      TaskID.getRepresentingCharacter(taskId.getTaskType()));
  result.append('-');
  result.append(NUMBER_FORMAT.format(partition));
  result.append(extension);
  return result.toString();
}
Example 3
Source File: LogOutputFormat.java    From 163-bigdate-note with GNU General Public License v3.0 5 votes vote down vote up 
public void write(K key, V value) throws IOException, InterruptedException {
    TaskID taskID = job.getTaskAttemptID().getTaskID();
    int partition = taskID.getId();
    String baseName = getFileBaseName(key, NUMBER_FORMAT.format(partition));
    RecordWriter<K, V> rw = this.recordWriter.get(baseName);
    if (rw == null) {
        rw = getBaseRecordWriter(job, baseName);
        this.recordWriter.put(baseName, rw);
    }
    rw.write(null, value);
}
Example 4
Source File: CustomFileNameFileOutputFormat.java    From aegisthus with Apache License 2.0 5 votes vote down vote up 
/**
 * Generate a unique filename, based on the task id, name, and extension
 * @param context the task that is calling this
 * @param name the base filename
 * @param extension the filename extension
 * @return a string like $name-[jobType]-$id$extension
 */
protected synchronized String getCustomFileName(TaskAttemptContext context,
        String name,
        String extension) {
    TaskID taskId = context.getTaskAttemptID().getTaskID();
    int partition = taskId.getId();
    return name + '-' + NUMBER_FORMAT.format(partition) + extension;
}
Example 5
Source File: FileOutputFormat.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up 
/**
 * Generate a unique filename, based on the task id, name, and extension
 * @param context the task that is calling this
 * @param name the base filename
 * @param extension the filename extension
 * @return a string like $name-[mr]-$id$extension
 */
public synchronized static String getUniqueFile(TaskAttemptContext context,
                                                String name,
                                                String extension) {
  TaskID taskId = context.getTaskAttemptID().getTaskID();
  int partition = taskId.getId();
  StringBuilder result = new StringBuilder();
  result.append(name);
  result.append('-');
  result.append(taskId.isMap() ? 'm' : 'r');
  result.append('-');
  result.append(NUMBER_FORMAT.format(partition));
  result.append(extension);
  return result.toString();
}
Example 6
Source File: SSTableOutputFormat.java    From aegisthus with Apache License 2.0 5 votes vote down vote up 
@Override
public synchronized String getCustomFileName(TaskAttemptContext context, String name, String extension) {
    TaskID taskId = context.getTaskAttemptID().getTaskID();
    int partition = taskId.getId();
    String sstableVersion = context.getConfiguration().get(Aegisthus.Feature.CONF_SSTABLE_VERSION);
    return context.getConfiguration().get(Aegisthus.Feature.CONF_DATASET, "keyspace-dataset")
            + "-" + sstableVersion
            + "-" + NUMBER_FORMAT.format(partition)
            + "-Data.db";
}
Example 7
Source File: FileOutputFormat.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
/**
 * Generate a unique filename, based on the task id, name, and extension
 * @param context the task that is calling this
 * @param name the base filename
 * @param extension the filename extension
 * @return a string like $name-[mr]-$id$extension
 */
public synchronized static String getUniqueFile(TaskAttemptContext context,
                                                String name,
                                                String extension) {
  TaskID taskId = context.getTaskAttemptID().getTaskID();
  int partition = taskId.getId();
  StringBuilder result = new StringBuilder();
  result.append(name);
  result.append('-');
  result.append(taskId.isMap() ? 'm' : 'r');
  result.append('-');
  result.append(NUMBER_FORMAT.format(partition));
  result.append(extension);
  return result.toString();
}
Example 8
Source File: HadoopFormatIO.java    From beam with Apache License 2.0 5 votes vote down vote up 
@ProcessElement
public void processElement(
    @Element KV<KeyT, ValueT> element,
    OutputReceiver<KV<Integer, KV<KeyT, ValueT>>> receiver,
    ProcessContext c) {

  Configuration config = c.sideInput(configView);

  TaskID taskID = createTaskIDForKV(element, config);
  int taskId = taskID.getId();
  receiver.output(KV.of(taskId, element));
}
Example 9
Source File: MneDurableOutputSession.java    From mnemonic with Apache License 2.0 5 votes vote down vote up 
protected String getUniqueName(String name, String extension) {
  int partition;
  
  NumberFormat numberFormat = NumberFormat.getInstance();
  numberFormat.setMinimumIntegerDigits(5);
  numberFormat.setGroupingUsed(false);
  
  if (null != getTaskAttemptContext()) {
    TaskID taskId = getTaskAttemptContext().getTaskAttemptID().getTaskID();
    partition = taskId.getId();
  } else {
    partition = getConfiguration().getInt(JobContext.TASK_PARTITION, -1);
  } 
  if (partition == -1) {
    throw new IllegalArgumentException("This method can only be called from an application");
  }
  
  String taskType = getConfiguration().getBoolean(JobContext.TASK_ISMAP, JobContext.DEFAULT_TASK_ISMAP) ? "m" : "r";
  
  StringBuilder result = new StringBuilder();
  result.append(name);
  result.append('-');
  result.append(taskType);
  result.append('-');
  result.append(numberFormat.format(partition));
  result.append(extension);
  return result.toString();
  
}
Example 10
Source File: ZombieJob.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Mask the job ID part in a {@link TaskAttemptID}.
 * 
 * @param attemptId
 *          raw {@link TaskAttemptID} read from trace
 * @return masked {@link TaskAttemptID} with empty {@link JobID}.
 */
private TaskAttemptID maskAttemptID(TaskAttemptID attemptId) {
  JobID jobId = new JobID();
  TaskType taskType = attemptId.getTaskType();
  TaskID taskId = attemptId.getTaskID();
  return new TaskAttemptID(jobId.getJtIdentifier(), jobId.getId(), taskType,
      taskId.getId(), attemptId.getId());
}
Example 11
Source File: ShuffleSchedulerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public synchronized void tipFailed(TaskID taskId) {
  if (!finishedMaps[taskId.getId()]) {
    finishedMaps[taskId.getId()] = true;
    if (--remainingMaps == 0) {
      notifyAll();
    }
    updateStatus();
  }
}
Example 12
Source File: ZombieJob.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Mask the job ID part in a {@link TaskAttemptID}.
 * 
 * @param attemptId
 *          raw {@link TaskAttemptID} read from trace
 * @return masked {@link TaskAttemptID} with empty {@link JobID}.
 */
private TaskAttemptID maskAttemptID(TaskAttemptID attemptId) {
  JobID jobId = new JobID();
  TaskType taskType = attemptId.getTaskType();
  TaskID taskId = attemptId.getTaskID();
  return new TaskAttemptID(jobId.getJtIdentifier(), jobId.getId(), taskType,
      taskId.getId(), attemptId.getId());
}
Example 13
Source File: ShuffleSchedulerImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public synchronized void tipFailed(TaskID taskId) {
  if (!finishedMaps[taskId.getId()]) {
    finishedMaps[taskId.getId()] = true;
    if (--remainingMaps == 0) {
      notifyAll();
    }
    updateStatus();
  }
}
Example 14
Source File: LogOutputFormat.java    From 163-bigdate-note with GNU General Public License v3.0 5 votes vote down vote up 
public void write(K key, V value) throws IOException, InterruptedException {
    TaskID taskID = job.getTaskAttemptID().getTaskID();
    int partition = taskID.getId();
    String baseName = getFileBaseName(key, NUMBER_FORMAT.format(partition));
    RecordWriter<K, V> rw = this.recordWriter.get(baseName);
    if (rw == null) {
        rw = getBaseRecordWriter(job, baseName);
        this.recordWriter.put(baseName, rw);
    }
    rw.write(null, value);
}
Example 15
Source File: LogOutputFormat.java    From 163-bigdate-note with GNU General Public License v3.0 5 votes vote down vote up 
public void write(K key, V value) throws IOException, InterruptedException {
    TaskID taskID = job.getTaskAttemptID().getTaskID();
    int partition = taskID.getId();
    String baseName = getFileBaseName(key, NUMBER_FORMAT.format(partition));
    RecordWriter<K, V> rw = this.recordWriter.get(baseName);
    if (rw == null) {
        rw = getBaseRecordWriter(job, baseName);
        this.recordWriter.put(baseName, rw);
    }
    rw.write(null, value);
}
Example 16
Source File: LogOutputFormat.java    From 163-bigdate-note with GNU General Public License v3.0 5 votes vote down vote up 
public void write(K key, V value) throws IOException, InterruptedException {
    TaskID taskID = job.getTaskAttemptID().getTaskID();
    int partition = taskID.getId();
    String baseName = getFileBaseName(key, NUMBER_FORMAT.format(partition));
    RecordWriter<K, V> rw = this.recordWriter.get(baseName);
    if (rw == null) {
        rw = getBaseRecordWriter(job, baseName);
        this.recordWriter.put(baseName, rw);
    }
    rw.write(null, value);
}
Example 17
Source File: ZombieJob.java    From RDFS with Apache License 2.0 3 votes vote down vote up 
/**
 * Mask the job ID part in a {@link TaskAttemptID}.
 * 
 * @param attemptId
 *          raw {@link TaskAttemptID} read from trace
 * @return masked {@link TaskAttemptID} with empty {@link JobID}.
 */
private TaskAttemptID maskAttemptID(TaskAttemptID attemptId) {
  JobID jobId = new JobID();
  TaskID taskId = attemptId.getTaskID();
  return new TaskAttemptID(jobId.getJtIdentifier(), jobId.getId(),
      attemptId.isMap(), taskId.getId(), attemptId.getId());
}
Example 18
Source File: ZombieJob.java    From big-c with Apache License 2.0 2 votes vote down vote up 
/**
 * Mask the job ID part in a {@link TaskID}.
 * 
 * @param taskId
 *          raw {@link TaskID} read from trace
 * @return masked {@link TaskID} with empty {@link JobID}.
 */
private TaskID maskTaskID(TaskID taskId) {
  JobID jobId = new JobID();
  TaskType taskType = taskId.getTaskType();
  return new TaskID(jobId, taskType, taskId.getId());
}
Example 19
Source File: ZombieJob.java    From hadoop with Apache License 2.0 2 votes vote down vote up 
/**
 * Mask the job ID part in a {@link TaskID}.
 * 
 * @param taskId
 *          raw {@link TaskID} read from trace
 * @return masked {@link TaskID} with empty {@link JobID}.
 */
private TaskID maskTaskID(TaskID taskId) {
  JobID jobId = new JobID();
  TaskType taskType = taskId.getTaskType();
  return new TaskID(jobId, taskType, taskId.getId());
}
Example 20
Source File: ZombieJob.java    From RDFS with Apache License 2.0 2 votes vote down vote up 
/**
 * Mask the job ID part in a {@link TaskID}.
 * 
 * @param taskId
 *          raw {@link TaskID} read from trace
 * @return masked {@link TaskID} with empty {@link JobID}.
 */
private TaskID maskTaskID(TaskID taskId) {
  JobID jobId = new JobID();
  return new TaskID(jobId, taskId.isMap(), taskId.getId());
}