Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationId#getId()

The following examples show how to use org.apache.hadoop.yarn.api.records.ApplicationId#getId() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   File: ShuffleHandler.java    License: Apache License 2.0 6 votes vote down vote up 
@Override
public void initializeApplication(ApplicationInitializationContext context) {

  String user = context.getUser();
  ApplicationId appId = context.getApplicationId();
  ByteBuffer secret = context.getApplicationDataForService();
  // TODO these bytes should be versioned
  try {
    Token<JobTokenIdentifier> jt = deserializeServiceData(secret);
     // TODO: Once SHuffle is out of NM, this can use MR APIs
    JobID jobId = new JobID(Long.toString(appId.getClusterTimestamp()), appId.getId());
    recordJobShuffleInfo(jobId, user, jt);
  } catch (IOException e) {
    LOG.error("Error during initApp", e);
    // TODO add API to AuxiliaryServices to report failures
  }
}
Example 2
Source Project: big-c   File: ShuffleHandler.java    License: Apache License 2.0 6 votes vote down vote up 
@Override
public void initializeApplication(ApplicationInitializationContext context) {

  String user = context.getUser();
  ApplicationId appId = context.getApplicationId();
  ByteBuffer secret = context.getApplicationDataForService();
  // TODO these bytes should be versioned
  try {
    Token<JobTokenIdentifier> jt = deserializeServiceData(secret);
     // TODO: Once SHuffle is out of NM, this can use MR APIs
    JobID jobId = new JobID(Long.toString(appId.getClusterTimestamp()), appId.getId());
    recordJobShuffleInfo(jobId, user, jt);
  } catch (IOException e) {
    LOG.error("Error during initApp", e);
    // TODO add API to AuxiliaryServices to report failures
  }
}
Example 3
Source Project: tez   File: ShuffleHandler.java    License: Apache License 2.0 6 votes vote down vote up 
@Override
public void initializeApplication(ApplicationInitializationContext context) {

  String user = context.getUser();
  ApplicationId appId = context.getApplicationId();
  ByteBuffer secret = context.getApplicationDataForService();
  // TODO these bytes should be versioned
  try {
    Token<JobTokenIdentifier> jt = deserializeServiceData(secret);
     // TODO: Once SHuffle is out of NM, this can use MR APIs
    JobID jobId = new JobID(Long.toString(appId.getClusterTimestamp()), appId.getId());
    recordJobShuffleInfo(jobId, user, jt);
  } catch (IOException e) {
    LOG.error("Error during initApp", e);
    // TODO add API to AuxiliaryServices to report failures
  }
}
Example 4
Source Project: spydra   File: HistoryLogUtils.java    License: Apache License 2.0 5 votes vote down vote up 
public static Optional<String> findHistoryFilePath(
    Iterator<LocatedFileStatus> listing, ApplicationId applicationId) {

  JobID jobId = new JobID(
      String.valueOf(applicationId.getClusterTimestamp()),
      applicationId.getId());

  List<LocatedFileStatus> jhistFiles = new ArrayList<>();
  // maybe this could work more nicely with some recursive glob and a filter
  try {
    jhistFiles = StreamSupport
        .stream(Spliterators.spliteratorUnknownSize(listing, Spliterator.NONNULL), false)
        .filter(fstatus -> fstatus.getPath().toString()
            .matches(".*" + jobId.toString() + ".*.jhist"))
        .collect(Collectors.toList());
  } catch (RemoteIteratorAdaptor.WrappedRemoteIteratorException wrie) {
    // We can't really do overly much at this point, as this is an error from the
    // underlying hadoop filesystem implementation. But we want to at least log this
    // separately from other conditions.
    logger.error("Retrieving remote listing failed", wrie);
  }

  if (jhistFiles.size() < 1) {
    logger.error("Could not locate a history file for parameters");
    return Optional.empty();
  } else if (jhistFiles.size() > 1) {
    logger.error("Found two or more matching files, will dump first");
  }

  return jhistFiles.stream()
      .findFirst()
      .map(x -> x.getPath().toString());
}
Example 5
Source Project: hadoop   File: ShuffleHandler.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void stopApplication(ApplicationTerminationContext context) {
  ApplicationId appId = context.getApplicationId();
  JobID jobId = new JobID(Long.toString(appId.getClusterTimestamp()), appId.getId());
  try {
    removeJobShuffleInfo(jobId);
  } catch (IOException e) {
    LOG.error("Error during stopApp", e);
    // TODO add API to AuxiliaryServices to report failures
  }
}
Example 6
Source Project: big-c   File: ShuffleHandler.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void stopApplication(ApplicationTerminationContext context) {
  ApplicationId appId = context.getApplicationId();
  JobID jobId = new JobID(Long.toString(appId.getClusterTimestamp()), appId.getId());
  try {
    removeJobShuffleInfo(jobId);
  } catch (IOException e) {
    LOG.error("Error during stopApp", e);
    // TODO add API to AuxiliaryServices to report failures
  }
}
Example 7
Source Project: spark-llap   File: HiveWarehouseDataReader.java    License: Apache License 2.0 5 votes vote down vote up 
protected TaskAttemptID getTaskAttemptID(LlapInputSplit split, JobConf conf) throws IOException {
  //Get pseudo-ApplicationId to submit task attempt from external client
  SubmitWorkInfo submitWorkInfo = SubmitWorkInfo.fromBytes(split.getPlanBytes());
  ApplicationId appId = submitWorkInfo.getFakeAppId();
  JobID jobId = new JobID(Long.toString(appId.getClusterTimestamp()), appId.getId());
  //Create TaskAttemptID from Spark TaskContext (TaskType doesn't matter)
  return new TaskAttemptID(new TaskID(jobId, TaskType.MAP, TaskContext.get().partitionId()), TaskContext.get().attemptNumber());
}
Example 8
Source Project: incubator-tez   File: TaskAttemptContextImpl.java    License: Apache License 2.0 5 votes vote down vote up 
public static org.apache.hadoop.mapred.TaskAttemptID 
  createMockTaskAttemptIDFromTezTaskAttemptId(TezTaskAttemptID tezTaId, boolean isMap) {
  TezVertexID vId = tezTaId.getTaskID().getVertexID();
  ApplicationId appId = vId.getDAGId().getApplicationId();
  return new org.apache.hadoop.mapred.TaskAttemptID(
      new org.apache.hadoop.mapred.TaskID(String.valueOf(appId.getClusterTimestamp())
          + String.valueOf(vId.getId()), appId.getId(),
          isMap ? TaskType.MAP : TaskType.REDUCE, tezTaId.getTaskID().getId()),
      tezTaId.getId());
}
Example 9
Source Project: incubator-tez   File: TaskAttemptContextImpl.java    License: Apache License 2.0 5 votes vote down vote up 
public static org.apache.hadoop.mapred.TaskID 
  createMockTaskAttemptIDFromTezTaskId(TezTaskID tezTaId, boolean isMap) {
  TezVertexID vId = tezTaId.getVertexID();
  ApplicationId appId = vId.getDAGId().getApplicationId();
  return new org.apache.hadoop.mapred.TaskID(String.valueOf(appId.getClusterTimestamp())
          + String.valueOf(vId.getId()), appId.getId(),
          isMap ? TaskType.MAP : TaskType.REDUCE, tezTaId.getId());
}
Example 10
Source Project: tez   File: TaskAttemptContextImpl.java    License: Apache License 2.0 5 votes vote down vote up 
public static org.apache.hadoop.mapred.TaskAttemptID 
  createMockTaskAttemptIDFromTezTaskAttemptId(TezTaskAttemptID tezTaId, boolean isMap) {
  TezVertexID vId = tezTaId.getTaskID().getVertexID();
  ApplicationId appId = vId.getDAGId().getApplicationId();
  return new org.apache.hadoop.mapred.TaskAttemptID(
      new org.apache.hadoop.mapred.TaskID(String.valueOf(appId.getClusterTimestamp())
          + String.valueOf(vId.getId()), appId.getId(),
          isMap ? TaskType.MAP : TaskType.REDUCE, tezTaId.getTaskID().getId()),
      tezTaId.getId());
}
Example 11
Source Project: tez   File: TaskAttemptContextImpl.java    License: Apache License 2.0 5 votes vote down vote up 
public static org.apache.hadoop.mapred.TaskID 
  createMockTaskAttemptIDFromTezTaskId(TezTaskID tezTaId, boolean isMap) {
  TezVertexID vId = tezTaId.getVertexID();
  ApplicationId appId = vId.getDAGId().getApplicationId();
  return new org.apache.hadoop.mapred.TaskID(String.valueOf(appId.getClusterTimestamp())
          + String.valueOf(vId.getId()), appId.getId(),
          isMap ? TaskType.MAP : TaskType.REDUCE, tezTaId.getId());
}
Example 12
Source Project: tez   File: ShuffleHandler.java    License: Apache License 2.0 5 votes vote down vote up 
@Override
public void stopApplication(ApplicationTerminationContext context) {
  ApplicationId appId = context.getApplicationId();
  JobID jobId = new JobID(Long.toString(appId.getClusterTimestamp()), appId.getId());
  try {
    removeJobShuffleInfo(jobId);
  } catch (IOException e) {
    LOG.error("Error during stopApp", e);
    // TODO add API to AuxiliaryServices to report failures
  }
}
Example 13
Source Project: hadoop   File: TypeConverter.java    License: Apache License 2.0 4 votes vote down vote up 
public static org.apache.hadoop.mapreduce.JobID fromYarn(ApplicationId appID) {
  String identifier = fromClusterTimeStamp(appID.getClusterTimestamp());
  return new org.apache.hadoop.mapred.JobID(identifier, appID.getId());
}
Example 14
Source Project: big-c   File: TypeConverter.java    License: Apache License 2.0 4 votes vote down vote up 
public static org.apache.hadoop.mapreduce.JobID fromYarn(ApplicationId appID) {
  String identifier = fromClusterTimeStamp(appID.getClusterTimestamp());
  return new org.apache.hadoop.mapred.JobID(identifier, appID.getId());
}