Java Code Examples for org.apache.hadoop.util.StringInterner#weakIntern()

public void readFields(DataInput in) throws IOException {
  this.taskid.readFields(in);
  this.progress = in.readFloat();
  this.state = StringInterner.weakIntern(Text.readString(in));
  this.startTime = in.readLong(); 
  this.finishTime = in.readLong();
  
  diagnostics = WritableUtils.readStringArray(in);
  counters = new Counters();
  counters.readFields(in);
  currentStatus = WritableUtils.readEnum(in, TIPStatus.class);
  if (currentStatus == TIPStatus.RUNNING) {
    int num = WritableUtils.readVInt(in);    
    for (int i = 0; i < num; i++) {
      TaskAttemptID t = new TaskAttemptID();
      t.readFields(in);
      runningAttempts.add(t);
    }
  } else if (currentStatus == TIPStatus.COMPLETE) {
    successfulAttempt.readFields(in);
  }
}
 

public static String getFrameworkClasspath(Configuration conf) {
  Map<String, String> environment = new HashMap<String, String>();

  TezYARNUtils.addToEnvironment(environment,
      Environment.CLASSPATH.name(),
      Environment.PWD.$(),
      File.pathSeparator);

  TezYARNUtils.addToEnvironment(environment,
      Environment.CLASSPATH.name(),
      Environment.PWD.$() + File.separator + "*",
      File.pathSeparator);

  // Add YARN/COMMON/HDFS jars and conf locations to path
  for (String c : conf.getStrings(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
    TezYARNUtils.addToEnvironment(environment, Environment.CLASSPATH.name(),
        c.trim(), File.pathSeparator);
  }
  return StringInterner.weakIntern(environment.get(Environment.CLASSPATH.name()));
}
 

public TaskSpec(
    String dagName, String vertexName,
    int vertexParallelism,
    ProcessorDescriptor processorDescriptor,
    List<InputSpec> inputSpecList, List<OutputSpec> outputSpecList,
    @Nullable List<GroupInputSpec> groupInputSpecList, Configuration taskConf) {
  Objects.requireNonNull(dagName, "dagName is null");
  Objects.requireNonNull(vertexName, "vertexName is null");
  Objects.requireNonNull(processorDescriptor, "processorDescriptor is null");
  Objects.requireNonNull(inputSpecList, "inputSpecList is null");
  Objects.requireNonNull(outputSpecList, "outputSpecList is null");
  this.taskAttemptId = null;
  this.dagName = StringInterner.weakIntern(dagName);
  this.vertexName = StringInterner.weakIntern(vertexName);
  this.processorDescriptor = processorDescriptor;
  this.inputSpecList = inputSpecList;
  this.outputSpecList = outputSpecList;
  this.groupInputSpecList = groupInputSpecList;
  this.vertexParallelism = vertexParallelism;
  this.taskConf = taskConf;
}
 

@Override
 public void reportDiagnosticInfo(TaskAttemptID taskAttemptID, String diagnosticInfo)
throws IOException {
   diagnosticInfo = StringInterner.weakIntern(diagnosticInfo);
   LOG.info("Diagnostics report from " + taskAttemptID.toString() + ": "
       + diagnosticInfo);

   org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
     TypeConverter.toYarn(taskAttemptID);
   taskHeartbeatHandler.progressing(attemptID);

   // This is mainly used for cases where we want to propagate exception traces
   // of tasks that fail.

   // This call exists as a hadoop mapreduce legacy wherein all changes in
   // counters/progress/phase/output-size are reported through statusUpdate()
   // call but not diagnosticInformation.
   context.getEventHandler().handle(
       new TaskAttemptDiagnosticsUpdateEvent(attemptID, diagnosticInfo));
 }
 

private void handleTaskFailedEvent(TaskFailedEvent event) {
  TaskInfo taskInfo = info.tasksMap.get(event.getTaskId());
  taskInfo.status = TaskStatus.State.FAILED.toString();
  taskInfo.finishTime = event.getFinishTime();
  taskInfo.error = StringInterner.weakIntern(event.getError());
  taskInfo.failedDueToAttemptId = event.getFailedAttemptID();
  taskInfo.counters = event.getCounters();
}
 

private void handleTaskAttemptFinishedEvent(TaskAttemptFinishedEvent event) {
  TaskInfo taskInfo = info.tasksMap.get(event.getTaskId());
  TaskAttemptInfo attemptInfo = 
    taskInfo.attemptsMap.get(event.getAttemptId());
  attemptInfo.finishTime = event.getFinishTime();
  attemptInfo.status = StringInterner.weakIntern(event.getTaskStatus());
  attemptInfo.state = StringInterner.weakIntern(event.getState());
  attemptInfo.counters = event.getCounters();
  attemptInfo.hostname = StringInterner.weakIntern(event.getHostname());
  info.completedTaskAttemptsMap.put(event.getAttemptId(), attemptInfo);
}
 

@Override
public void readFields(DataInput in) throws IOException {
  taskAttemptId = TezTaskAttemptID.readTezTaskAttemptID(in);
  dagName = StringInterner.weakIntern(in.readUTF());
  vertexName = StringInterner.weakIntern(in.readUTF());
  // TODO TEZ-305 convert this to PB
  processorDescriptor = new ProcessorDescriptor();
  processorDescriptor.readFields(in);
  int numInputSpecs = in.readInt();
  inputSpecList = new ArrayList<InputSpec>(numInputSpecs);
  for (int i = 0; i < numInputSpecs; i++) {
    InputSpec inputSpec = new InputSpec();
    inputSpec.readFields(in);
    inputSpecList.add(inputSpec);
  }
  int numOutputSpecs = in.readInt();
  outputSpecList = new ArrayList<OutputSpec>(numOutputSpecs);
  for (int i = 0; i < numOutputSpecs; i++) {
    OutputSpec outputSpec = new OutputSpec();
    outputSpec.readFields(in);
    outputSpecList.add(outputSpec);
  }
  boolean hasGroupInputs = in.readBoolean();
  if (hasGroupInputs) {
    int numGroups = in.readInt();
    groupInputSpecList = Lists.newArrayListWithCapacity(numGroups);
    for (int i=0; i<numGroups; ++i) {
      GroupInputSpec group = new GroupInputSpec();
      group.readFields(in);
      groupInputSpecList.add(group);
    }
  }
}
 

@Override
public void readFields(DataInput in) throws IOException {
  taskAttemptId = TezTaskAttemptID.readTezTaskAttemptID(in);
  dagName = StringInterner.weakIntern(in.readUTF());
  vertexName = StringInterner.weakIntern(in.readUTF());
  vertexParallelism = in.readInt();
  // TODO TEZ-305 convert this to PB
  processorDescriptor = new ProcessorDescriptor();
  processorDescriptor.readFields(in);
  int numInputSpecs = in.readInt();
  inputSpecList = new ArrayList<InputSpec>(numInputSpecs);
  for (int i = 0; i < numInputSpecs; i++) {
    InputSpec inputSpec = new InputSpec();
    inputSpec.readFields(in);
    inputSpecList.add(inputSpec);
  }
  int numOutputSpecs = in.readInt();
  outputSpecList = new ArrayList<OutputSpec>(numOutputSpecs);
  for (int i = 0; i < numOutputSpecs; i++) {
    OutputSpec outputSpec = new OutputSpec();
    outputSpec.readFields(in);
    outputSpecList.add(outputSpec);
  }
  boolean hasGroupInputs = in.readBoolean();
  if (hasGroupInputs) {
    int numGroups = in.readInt();
    groupInputSpecList = Lists.newArrayListWithCapacity(numGroups);
    for (int i=0; i<numGroups; ++i) {
      GroupInputSpec group = new GroupInputSpec();
      group.readFields(in);
      groupInputSpecList.add(group);
    }
  }
  boolean hasVertexConf = in.readBoolean();
  if (hasVertexConf) {
    taskConf = new Configuration(false);
    taskConf.readFields(in);
  }
}
 

private void handleTaskAttemptFinishedEvent(TaskAttemptFinishedEvent event) {
  TaskInfo taskInfo = info.tasksMap.get(event.getTaskId());
  TaskAttemptInfo attemptInfo = 
    taskInfo.attemptsMap.get(event.getAttemptId());
  attemptInfo.finishTime = event.getFinishTime();
  attemptInfo.status = StringInterner.weakIntern(event.getTaskStatus());
  attemptInfo.state = StringInterner.weakIntern(event.getState());
  attemptInfo.counters = event.getCounters();
  attemptInfo.hostname = StringInterner.weakIntern(event.getHostname());
  info.completedTaskAttemptsMap.put(event.getAttemptId(), attemptInfo);
}
 

public void readFields(DataInput in) throws IOException {
  jobFile = StringInterner.weakIntern(Text.readString(in));
  taskId = TaskAttemptID.read(in);
  partition = in.readInt();
  numSlotsRequired = in.readInt();
  taskStatus.readFields(in);
  skipRanges.readFields(in);
  currentRecIndexIterator = skipRanges.skipRangeIterator();
  currentRecStartIndex = currentRecIndexIterator.next();
  skipping = in.readBoolean();
  jobCleanup = in.readBoolean();
  if (jobCleanup) {
    jobRunStateForCleanup = 
      WritableUtils.readEnum(in, JobStatus.State.class);
  }
  jobSetup = in.readBoolean();
  writeSkipRecs = in.readBoolean();
  taskCleanup = in.readBoolean();
  if (taskCleanup) {
    setPhase(TaskStatus.Phase.CLEANUP);
  }
  user = StringInterner.weakIntern(Text.readString(in));
  int len = in.readInt();
  encryptedSpillKey = new byte[len];
  extraData.readFields(in);
  in.readFully(encryptedSpillKey);
}
 

public void readFields(DataInput in) throws IOException {
  numActiveTrackers = in.readInt();
  int numTrackerNames = in.readInt();
  if (numTrackerNames > 0) {
    for (int i = 0; i < numTrackerNames; i++) {
      String name = StringInterner.weakIntern(Text.readString(in));
      activeTrackers.add(name);
    }
  }
  numBlacklistedTrackers = in.readInt();
  int blackListTrackerInfoSize = in.readInt();
  if(blackListTrackerInfoSize > 0) {
    for (int i = 0; i < blackListTrackerInfoSize; i++) {
      BlackListInfo info = new BlackListInfo();
      info.readFields(in);
      blacklistedTrackersInfo.add(info);
    }
  }
  numExcludedNodes = in.readInt();
  ttExpiryInterval = in.readLong();
  map_tasks = in.readInt();
  reduce_tasks = in.readInt();
  max_map_tasks = in.readInt();
  max_reduce_tasks = in.readInt();
  status = WritableUtils.readEnum(in, JobTrackerStatus.class);
  grayListedTrackers = in.readInt();
}
 

private Class<?> readClass(DataInput in) throws IOException {
  String className = StringInterner.weakIntern(Text.readString(in));
  try {
    return conf.getClassByName(className);
  } catch (ClassNotFoundException e) {
    throw new RuntimeException("readObject can't find class", e);
  }
}
 

private void handleAMStartedEvent(AMStartedEvent event) {
  AMInfo amInfo = new AMInfo();
  amInfo.appAttemptId = event.getAppAttemptId();
  amInfo.startTime = event.getStartTime();
  amInfo.containerId = event.getContainerId();
  amInfo.nodeManagerHost = StringInterner.weakIntern(event.getNodeManagerHost());
  amInfo.nodeManagerPort = event.getNodeManagerPort();
  amInfo.nodeManagerHttpPort = event.getNodeManagerHttpPort();
  if (info.amInfos == null) {
    info.amInfos = new LinkedList<AMInfo>();
  }
  info.amInfos.add(amInfo);
  info.latestAmInfo = amInfo;
}
 

private void handleTaskAttemptStartedEvent(TaskAttemptStartedEvent event) {
  TaskAttemptID attemptId = event.getTaskAttemptId();
  TaskInfo taskInfo = info.tasksMap.get(event.getTaskId());
  
  TaskAttemptInfo attemptInfo = new TaskAttemptInfo();
  attemptInfo.startTime = event.getStartTime();
  attemptInfo.attemptId = event.getTaskAttemptId();
  attemptInfo.httpPort = event.getHttpPort();
  attemptInfo.trackerName = StringInterner.weakIntern(event.getTrackerName());
  attemptInfo.taskType = event.getTaskType();
  attemptInfo.shufflePort = event.getShufflePort();
  attemptInfo.containerId = event.getContainerId();
  
  taskInfo.attemptsMap.put(attemptId, attemptInfo);
}
 

private Class<?> readClass(DataInput in) throws IOException {
  String className = StringInterner.weakIntern(Text.readString(in));
  try {
    return conf.getClassByName(className);
  } catch (ClassNotFoundException e) {
    throw new RuntimeException("readObject can't find class", e);
  }
}
 

@Override
public String getAssignedContainerMgrAddress() {
  readLock.lock();
  try {
    return container == null ? null : StringInterner.weakIntern(container
      .getNodeId().toString());
  } finally {
    readLock.unlock();
  }
}
 

public AbstractCounterGroup(String name, String displayName,
                            Limits limits) {
  this.name = StringInterner.weakIntern(name);
  this.displayName = StringInterner.weakIntern(displayName);
  this.limits = limits;
}
 

DagInfo(JSONObject jsonObject) throws JSONException {
  super(jsonObject);

  vertexNameMap = Maps.newHashMap();
  vertexNameIDMapping = new DualHashBidiMap<>();
  edgeInfoMap = Maps.newHashMap();
  basicVertexInfoMap = Maps.newHashMap();
  containerMapping = LinkedHashMultimap.create();

  Preconditions.checkArgument(jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase
      (Constants.TEZ_DAG_ID));

  dagId = StringInterner.weakIntern(jsonObject.getString(Constants.ENTITY));

  //Parse additional Info
  JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO);

  long sTime = otherInfoNode.optLong(Constants.START_TIME);
  long eTime= otherInfoNode.optLong(Constants.FINISH_TIME);
  userName = otherInfoNode.optString(Constants.USER);
  if (eTime < sTime) {
    LOG.warn("DAG has got wrong start/end values. "
        + "startTime=" + sTime + ", endTime=" + eTime + ". Will check "
        + "timestamps in DAG started/finished events");

    // Check if events DAG_STARTED, DAG_FINISHED can be made use of
    for(Event event : eventList) {
      switch (HistoryEventType.valueOf(event.getType())) {
      case DAG_STARTED:
        sTime = event.getAbsoluteTime();
        break;
      case DAG_FINISHED:
        eTime = event.getAbsoluteTime();
        break;
      default:
        break;
      }
    }

    if (eTime < sTime) {
      LOG.warn("DAG has got wrong start/end values in events as well. "
          + "startTime=" + sTime + ", endTime=" + eTime);
    }
  }
  startTime = sTime;
  endTime = eTime;

  //TODO: Not getting populated correctly for lots of jobs.  Verify
  submitTime = otherInfoNode.optLong(Constants.START_REQUESTED_TIME);
  diagnostics = otherInfoNode.optString(Constants.DIAGNOSTICS);
  failedTasks = otherInfoNode.optInt(Constants.NUM_FAILED_TASKS);
  JSONObject dagPlan = otherInfoNode.optJSONObject(Constants.DAG_PLAN);
  name = StringInterner.weakIntern((dagPlan != null) ? (dagPlan.optString(Constants.DAG_NAME)) : null);
  if (dagPlan != null) {
    JSONArray vertices = dagPlan.optJSONArray(Constants.VERTICES);
    if (vertices != null) {
      numVertices = vertices.length();
    } else {
      numVertices = 0;
    }
    parseDAGPlan(dagPlan);
  } else {
    numVertices = 0;
  }
  status = StringInterner.weakIntern(otherInfoNode.optString(Constants.STATUS));

  //parse name id mapping
  JSONObject vertexIDMappingJson = otherInfoNode.optJSONObject(Constants.VERTEX_NAME_ID_MAPPING);
  if (vertexIDMappingJson != null) {
    //get vertex name
    for (Map.Entry<String, BasicVertexInfo> entry : basicVertexInfoMap.entrySet()) {
      String vertexId = vertexIDMappingJson.optString(entry.getKey());
      //vertexName --> vertexId
      vertexNameIDMapping.put(entry.getKey(), vertexId);
    }
  }
}
 

public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan,
    String vertexName, Configuration conf, EventHandler eventHandler,
    TaskAttemptListener taskAttemptListener, Clock clock,
    TaskHeartbeatHandler thh, boolean commitVertexOutputs,
    AppContext appContext, VertexLocationHint vertexLocationHint,
    Map<String, VertexGroupInfo> dagVertexGroups, JavaProfilerOptions profilerOpts) {
  this.vertexId = vertexId;
  this.vertexPlan = vertexPlan;
  this.vertexName = StringInterner.weakIntern(vertexName);
  this.conf = conf;
  this.clock = clock;
  this.appContext = appContext;
  this.commitVertexOutputs = commitVertexOutputs;

  this.taskAttemptListener = taskAttemptListener;
  this.taskHeartbeatHandler = thh;
  this.eventHandler = eventHandler;
  ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
  this.readLock = readWriteLock.readLock();
  this.writeLock = readWriteLock.writeLock();

  if (LOG.isDebugEnabled()) {
    logLocationHints(this.vertexName, vertexLocationHint);
  }
  setTaskLocationHints(vertexLocationHint);
  
  this.dagUgi = appContext.getCurrentDAG().getDagUGI();

  this.taskResource = DagTypeConverters
      .createResourceRequestFromTaskConfig(vertexPlan.getTaskConfig());
  this.processorDescriptor = DagTypeConverters
      .convertProcessorDescriptorFromDAGPlan(vertexPlan
          .getProcessorDescriptor());
  this.localResources = DagTypeConverters
      .createLocalResourceMapFromDAGPlan(vertexPlan.getTaskConfig()
          .getLocalResourceList());
  this.localResources.putAll(appContext.getSessionResources());
  this.environment = DagTypeConverters
      .createEnvironmentMapFromDAGPlan(vertexPlan.getTaskConfig()
          .getEnvironmentSettingList());
  this.javaOpts = vertexPlan.getTaskConfig().hasJavaOpts() ? vertexPlan
      .getTaskConfig().getJavaOpts() : null;
  this.profilerOpts = profilerOpts;
  this.containerContext = new ContainerContext(this.localResources,
      appContext.getCurrentDAG().getCredentials(), this.environment, this.javaOpts, this);

  if (vertexPlan.getInputsCount() > 0) {
    setAdditionalInputs(vertexPlan.getInputsList());
  }
  if (vertexPlan.getOutputsCount() > 0) {
    setAdditionalOutputs(vertexPlan.getOutputsList());
  }

  // Setup the initial parallelism early. This may be changed after
  // initialization or on a setParallelism call.
  this.numTasks = vertexPlan.getTaskConfig().getNumTasks();

  this.dagVertexGroups = dagVertexGroups;

  logIdentifier =  this.getVertexId() + " [" + this.getName() + "]";
  // This "this leak" is okay because the retained pointer is in an
  //  instance variable.
  stateMachine = stateMachineFactory.make(this);
}
 

TaskInfo(JSONObject jsonObject) throws JSONException {
  super(jsonObject);

  Preconditions.checkArgument(
      jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase
          (Constants.TEZ_TASK_ID));

  taskId = StringInterner.weakIntern(jsonObject.optString(Constants.ENTITY));

  //Parse additional Info
  final JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO);

  long sTime = otherInfoNode.optLong(Constants.START_TIME);
  long eTime = otherInfoNode.optLong(Constants.FINISH_TIME);
  if (eTime < sTime) {
    LOG.warn("Task has got wrong start/end values. "
        + "startTime=" + sTime + ", endTime=" + eTime + ". Will check "
        + "timestamps in DAG started/finished events");

    // Check if events TASK_STARTED, TASK_FINISHED can be made use of
    for(Event event : eventList) {
      switch (HistoryEventType.valueOf(event.getType())) {
      case TASK_STARTED:
        sTime = event.getAbsoluteTime();
        break;
      case TASK_FINISHED:
        eTime = event.getAbsoluteTime();
        break;
      default:
        break;
      }
    }

    if (eTime < sTime) {
      LOG.warn("Task has got wrong start/end values in events as well. "
          + "startTime=" + sTime + ", endTime=" + eTime);
    }
  }
  startTime = sTime;
  endTime = eTime;

  diagnostics = otherInfoNode.optString(Constants.DIAGNOSTICS);
  successfulAttemptId = StringInterner.weakIntern(
      otherInfoNode.optString(Constants.SUCCESSFUL_ATTEMPT_ID));
  scheduledTime = otherInfoNode.optLong(Constants.SCHEDULED_TIME);
  status = StringInterner.weakIntern(otherInfoNode.optString(Constants.STATUS));
}