Java Code Examples for org.apache.hadoop.util.StringInterner#weakIntern()

The following examples show how to use org.apache.hadoop.util.StringInterner#weakIntern() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TaskReport.java    From big-c with Apache License 2.0 6 votes vote down vote up
public void readFields(DataInput in) throws IOException {
  this.taskid.readFields(in);
  this.progress = in.readFloat();
  this.state = StringInterner.weakIntern(Text.readString(in));
  this.startTime = in.readLong(); 
  this.finishTime = in.readLong();
  
  diagnostics = WritableUtils.readStringArray(in);
  counters = new Counters();
  counters.readFields(in);
  currentStatus = WritableUtils.readEnum(in, TIPStatus.class);
  if (currentStatus == TIPStatus.RUNNING) {
    int num = WritableUtils.readVInt(in);    
    for (int i = 0; i < num; i++) {
      TaskAttemptID t = new TaskAttemptID();
      t.readFields(in);
      runningAttempts.add(t);
    }
  } else if (currentStatus == TIPStatus.COMPLETE) {
    successfulAttempt.readFields(in);
  }
}
 
Example 2
Source File: TezYARNUtils.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
public static String getFrameworkClasspath(Configuration conf) {
  Map<String, String> environment = new HashMap<String, String>();

  TezYARNUtils.addToEnvironment(environment,
      Environment.CLASSPATH.name(),
      Environment.PWD.$(),
      File.pathSeparator);

  TezYARNUtils.addToEnvironment(environment,
      Environment.CLASSPATH.name(),
      Environment.PWD.$() + File.separator + "*",
      File.pathSeparator);

  // Add YARN/COMMON/HDFS jars and conf locations to path
  for (String c : conf.getStrings(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
    TezYARNUtils.addToEnvironment(environment, Environment.CLASSPATH.name(),
        c.trim(), File.pathSeparator);
  }
  return StringInterner.weakIntern(environment.get(Environment.CLASSPATH.name()));
}
 
Example 3
Source File: TaskSpec.java    From tez with Apache License 2.0 6 votes vote down vote up
public TaskSpec(
    String dagName, String vertexName,
    int vertexParallelism,
    ProcessorDescriptor processorDescriptor,
    List<InputSpec> inputSpecList, List<OutputSpec> outputSpecList,
    @Nullable List<GroupInputSpec> groupInputSpecList, Configuration taskConf) {
  Objects.requireNonNull(dagName, "dagName is null");
  Objects.requireNonNull(vertexName, "vertexName is null");
  Objects.requireNonNull(processorDescriptor, "processorDescriptor is null");
  Objects.requireNonNull(inputSpecList, "inputSpecList is null");
  Objects.requireNonNull(outputSpecList, "outputSpecList is null");
  this.taskAttemptId = null;
  this.dagName = StringInterner.weakIntern(dagName);
  this.vertexName = StringInterner.weakIntern(vertexName);
  this.processorDescriptor = processorDescriptor;
  this.inputSpecList = inputSpecList;
  this.outputSpecList = outputSpecList;
  this.groupInputSpecList = groupInputSpecList;
  this.vertexParallelism = vertexParallelism;
  this.taskConf = taskConf;
}
 
Example 4
Source File: TaskAttemptListenerImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
 public void reportDiagnosticInfo(TaskAttemptID taskAttemptID, String diagnosticInfo)
throws IOException {
   diagnosticInfo = StringInterner.weakIntern(diagnosticInfo);
   LOG.info("Diagnostics report from " + taskAttemptID.toString() + ": "
       + diagnosticInfo);

   org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
     TypeConverter.toYarn(taskAttemptID);
   taskHeartbeatHandler.progressing(attemptID);

   // This is mainly used for cases where we want to propagate exception traces
   // of tasks that fail.

   // This call exists as a hadoop mapreduce legacy wherein all changes in
   // counters/progress/phase/output-size are reported through statusUpdate()
   // call but not diagnosticInformation.
   context.getEventHandler().handle(
       new TaskAttemptDiagnosticsUpdateEvent(attemptID, diagnosticInfo));
 }
 
Example 5
Source File: JobHistoryParser.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void handleTaskFailedEvent(TaskFailedEvent event) {
  TaskInfo taskInfo = info.tasksMap.get(event.getTaskId());
  taskInfo.status = TaskStatus.State.FAILED.toString();
  taskInfo.finishTime = event.getFinishTime();
  taskInfo.error = StringInterner.weakIntern(event.getError());
  taskInfo.failedDueToAttemptId = event.getFailedAttemptID();
  taskInfo.counters = event.getCounters();
}
 
Example 6
Source File: JobHistoryParser.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void handleTaskAttemptFinishedEvent(TaskAttemptFinishedEvent event) {
  TaskInfo taskInfo = info.tasksMap.get(event.getTaskId());
  TaskAttemptInfo attemptInfo = 
    taskInfo.attemptsMap.get(event.getAttemptId());
  attemptInfo.finishTime = event.getFinishTime();
  attemptInfo.status = StringInterner.weakIntern(event.getTaskStatus());
  attemptInfo.state = StringInterner.weakIntern(event.getState());
  attemptInfo.counters = event.getCounters();
  attemptInfo.hostname = StringInterner.weakIntern(event.getHostname());
  info.completedTaskAttemptsMap.put(event.getAttemptId(), attemptInfo);
}
 
Example 7
Source File: TaskSpec.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@Override
public void readFields(DataInput in) throws IOException {
  taskAttemptId = TezTaskAttemptID.readTezTaskAttemptID(in);
  dagName = StringInterner.weakIntern(in.readUTF());
  vertexName = StringInterner.weakIntern(in.readUTF());
  // TODO TEZ-305 convert this to PB
  processorDescriptor = new ProcessorDescriptor();
  processorDescriptor.readFields(in);
  int numInputSpecs = in.readInt();
  inputSpecList = new ArrayList<InputSpec>(numInputSpecs);
  for (int i = 0; i < numInputSpecs; i++) {
    InputSpec inputSpec = new InputSpec();
    inputSpec.readFields(in);
    inputSpecList.add(inputSpec);
  }
  int numOutputSpecs = in.readInt();
  outputSpecList = new ArrayList<OutputSpec>(numOutputSpecs);
  for (int i = 0; i < numOutputSpecs; i++) {
    OutputSpec outputSpec = new OutputSpec();
    outputSpec.readFields(in);
    outputSpecList.add(outputSpec);
  }
  boolean hasGroupInputs = in.readBoolean();
  if (hasGroupInputs) {
    int numGroups = in.readInt();
    groupInputSpecList = Lists.newArrayListWithCapacity(numGroups);
    for (int i=0; i<numGroups; ++i) {
      GroupInputSpec group = new GroupInputSpec();
      group.readFields(in);
      groupInputSpecList.add(group);
    }
  }
}
 
Example 8
Source File: TaskSpec.java    From tez with Apache License 2.0 5 votes vote down vote up
@Override
public void readFields(DataInput in) throws IOException {
  taskAttemptId = TezTaskAttemptID.readTezTaskAttemptID(in);
  dagName = StringInterner.weakIntern(in.readUTF());
  vertexName = StringInterner.weakIntern(in.readUTF());
  vertexParallelism = in.readInt();
  // TODO TEZ-305 convert this to PB
  processorDescriptor = new ProcessorDescriptor();
  processorDescriptor.readFields(in);
  int numInputSpecs = in.readInt();
  inputSpecList = new ArrayList<InputSpec>(numInputSpecs);
  for (int i = 0; i < numInputSpecs; i++) {
    InputSpec inputSpec = new InputSpec();
    inputSpec.readFields(in);
    inputSpecList.add(inputSpec);
  }
  int numOutputSpecs = in.readInt();
  outputSpecList = new ArrayList<OutputSpec>(numOutputSpecs);
  for (int i = 0; i < numOutputSpecs; i++) {
    OutputSpec outputSpec = new OutputSpec();
    outputSpec.readFields(in);
    outputSpecList.add(outputSpec);
  }
  boolean hasGroupInputs = in.readBoolean();
  if (hasGroupInputs) {
    int numGroups = in.readInt();
    groupInputSpecList = Lists.newArrayListWithCapacity(numGroups);
    for (int i=0; i<numGroups; ++i) {
      GroupInputSpec group = new GroupInputSpec();
      group.readFields(in);
      groupInputSpecList.add(group);
    }
  }
  boolean hasVertexConf = in.readBoolean();
  if (hasVertexConf) {
    taskConf = new Configuration(false);
    taskConf.readFields(in);
  }
}
 
Example 9
Source File: JobHistoryParser.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void handleTaskAttemptFinishedEvent(TaskAttemptFinishedEvent event) {
  TaskInfo taskInfo = info.tasksMap.get(event.getTaskId());
  TaskAttemptInfo attemptInfo = 
    taskInfo.attemptsMap.get(event.getAttemptId());
  attemptInfo.finishTime = event.getFinishTime();
  attemptInfo.status = StringInterner.weakIntern(event.getTaskStatus());
  attemptInfo.state = StringInterner.weakIntern(event.getState());
  attemptInfo.counters = event.getCounters();
  attemptInfo.hostname = StringInterner.weakIntern(event.getHostname());
  info.completedTaskAttemptsMap.put(event.getAttemptId(), attemptInfo);
}
 
Example 10
Source File: Task.java    From big-c with Apache License 2.0 5 votes vote down vote up
public void readFields(DataInput in) throws IOException {
  jobFile = StringInterner.weakIntern(Text.readString(in));
  taskId = TaskAttemptID.read(in);
  partition = in.readInt();
  numSlotsRequired = in.readInt();
  taskStatus.readFields(in);
  skipRanges.readFields(in);
  currentRecIndexIterator = skipRanges.skipRangeIterator();
  currentRecStartIndex = currentRecIndexIterator.next();
  skipping = in.readBoolean();
  jobCleanup = in.readBoolean();
  if (jobCleanup) {
    jobRunStateForCleanup = 
      WritableUtils.readEnum(in, JobStatus.State.class);
  }
  jobSetup = in.readBoolean();
  writeSkipRecs = in.readBoolean();
  taskCleanup = in.readBoolean();
  if (taskCleanup) {
    setPhase(TaskStatus.Phase.CLEANUP);
  }
  user = StringInterner.weakIntern(Text.readString(in));
  int len = in.readInt();
  encryptedSpillKey = new byte[len];
  extraData.readFields(in);
  in.readFully(encryptedSpillKey);
}
 
Example 11
Source File: ClusterStatus.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public void readFields(DataInput in) throws IOException {
  numActiveTrackers = in.readInt();
  int numTrackerNames = in.readInt();
  if (numTrackerNames > 0) {
    for (int i = 0; i < numTrackerNames; i++) {
      String name = StringInterner.weakIntern(Text.readString(in));
      activeTrackers.add(name);
    }
  }
  numBlacklistedTrackers = in.readInt();
  int blackListTrackerInfoSize = in.readInt();
  if(blackListTrackerInfoSize > 0) {
    for (int i = 0; i < blackListTrackerInfoSize; i++) {
      BlackListInfo info = new BlackListInfo();
      info.readFields(in);
      blacklistedTrackersInfo.add(info);
    }
  }
  numExcludedNodes = in.readInt();
  ttExpiryInterval = in.readLong();
  map_tasks = in.readInt();
  reduce_tasks = in.readInt();
  max_map_tasks = in.readInt();
  max_reduce_tasks = in.readInt();
  status = WritableUtils.readEnum(in, JobTrackerStatus.class);
  grayListedTrackers = in.readInt();
}
 
Example 12
Source File: TaggedInputSplit.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private Class<?> readClass(DataInput in) throws IOException {
  String className = StringInterner.weakIntern(Text.readString(in));
  try {
    return conf.getClassByName(className);
  } catch (ClassNotFoundException e) {
    throw new RuntimeException("readObject can't find class", e);
  }
}
 
Example 13
Source File: JobHistoryParser.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void handleAMStartedEvent(AMStartedEvent event) {
  AMInfo amInfo = new AMInfo();
  amInfo.appAttemptId = event.getAppAttemptId();
  amInfo.startTime = event.getStartTime();
  amInfo.containerId = event.getContainerId();
  amInfo.nodeManagerHost = StringInterner.weakIntern(event.getNodeManagerHost());
  amInfo.nodeManagerPort = event.getNodeManagerPort();
  amInfo.nodeManagerHttpPort = event.getNodeManagerHttpPort();
  if (info.amInfos == null) {
    info.amInfos = new LinkedList<AMInfo>();
  }
  info.amInfos.add(amInfo);
  info.latestAmInfo = amInfo;
}
 
Example 14
Source File: JobHistoryParser.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void handleTaskAttemptStartedEvent(TaskAttemptStartedEvent event) {
  TaskAttemptID attemptId = event.getTaskAttemptId();
  TaskInfo taskInfo = info.tasksMap.get(event.getTaskId());
  
  TaskAttemptInfo attemptInfo = new TaskAttemptInfo();
  attemptInfo.startTime = event.getStartTime();
  attemptInfo.attemptId = event.getTaskAttemptId();
  attemptInfo.httpPort = event.getHttpPort();
  attemptInfo.trackerName = StringInterner.weakIntern(event.getTrackerName());
  attemptInfo.taskType = event.getTaskType();
  attemptInfo.shufflePort = event.getShufflePort();
  attemptInfo.containerId = event.getContainerId();
  
  taskInfo.attemptsMap.put(attemptId, attemptInfo);
}
 
Example 15
Source File: TaggedInputSplit.java    From big-c with Apache License 2.0 5 votes vote down vote up
private Class<?> readClass(DataInput in) throws IOException {
  String className = StringInterner.weakIntern(Text.readString(in));
  try {
    return conf.getClassByName(className);
  } catch (ClassNotFoundException e) {
    throw new RuntimeException("readObject can't find class", e);
  }
}
 
Example 16
Source File: TaskAttemptImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public String getAssignedContainerMgrAddress() {
  readLock.lock();
  try {
    return container == null ? null : StringInterner.weakIntern(container
      .getNodeId().toString());
  } finally {
    readLock.unlock();
  }
}
 
Example 17
Source File: AbstractCounterGroup.java    From tez with Apache License 2.0 4 votes vote down vote up
public AbstractCounterGroup(String name, String displayName,
                            Limits limits) {
  this.name = StringInterner.weakIntern(name);
  this.displayName = StringInterner.weakIntern(displayName);
  this.limits = limits;
}
 
Example 18
Source File: DagInfo.java    From tez with Apache License 2.0 4 votes vote down vote up
DagInfo(JSONObject jsonObject) throws JSONException {
  super(jsonObject);

  vertexNameMap = Maps.newHashMap();
  vertexNameIDMapping = new DualHashBidiMap<>();
  edgeInfoMap = Maps.newHashMap();
  basicVertexInfoMap = Maps.newHashMap();
  containerMapping = LinkedHashMultimap.create();

  Preconditions.checkArgument(jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase
      (Constants.TEZ_DAG_ID));

  dagId = StringInterner.weakIntern(jsonObject.getString(Constants.ENTITY));

  //Parse additional Info
  JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO);

  long sTime = otherInfoNode.optLong(Constants.START_TIME);
  long eTime= otherInfoNode.optLong(Constants.FINISH_TIME);
  userName = otherInfoNode.optString(Constants.USER);
  if (eTime < sTime) {
    LOG.warn("DAG has got wrong start/end values. "
        + "startTime=" + sTime + ", endTime=" + eTime + ". Will check "
        + "timestamps in DAG started/finished events");

    // Check if events DAG_STARTED, DAG_FINISHED can be made use of
    for(Event event : eventList) {
      switch (HistoryEventType.valueOf(event.getType())) {
      case DAG_STARTED:
        sTime = event.getAbsoluteTime();
        break;
      case DAG_FINISHED:
        eTime = event.getAbsoluteTime();
        break;
      default:
        break;
      }
    }

    if (eTime < sTime) {
      LOG.warn("DAG has got wrong start/end values in events as well. "
          + "startTime=" + sTime + ", endTime=" + eTime);
    }
  }
  startTime = sTime;
  endTime = eTime;

  //TODO: Not getting populated correctly for lots of jobs.  Verify
  submitTime = otherInfoNode.optLong(Constants.START_REQUESTED_TIME);
  diagnostics = otherInfoNode.optString(Constants.DIAGNOSTICS);
  failedTasks = otherInfoNode.optInt(Constants.NUM_FAILED_TASKS);
  JSONObject dagPlan = otherInfoNode.optJSONObject(Constants.DAG_PLAN);
  name = StringInterner.weakIntern((dagPlan != null) ? (dagPlan.optString(Constants.DAG_NAME)) : null);
  if (dagPlan != null) {
    JSONArray vertices = dagPlan.optJSONArray(Constants.VERTICES);
    if (vertices != null) {
      numVertices = vertices.length();
    } else {
      numVertices = 0;
    }
    parseDAGPlan(dagPlan);
  } else {
    numVertices = 0;
  }
  status = StringInterner.weakIntern(otherInfoNode.optString(Constants.STATUS));

  //parse name id mapping
  JSONObject vertexIDMappingJson = otherInfoNode.optJSONObject(Constants.VERTEX_NAME_ID_MAPPING);
  if (vertexIDMappingJson != null) {
    //get vertex name
    for (Map.Entry<String, BasicVertexInfo> entry : basicVertexInfoMap.entrySet()) {
      String vertexId = vertexIDMappingJson.optString(entry.getKey());
      //vertexName --> vertexId
      vertexNameIDMapping.put(entry.getKey(), vertexId);
    }
  }
}
 
Example 19
Source File: VertexImpl.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
public VertexImpl(TezVertexID vertexId, VertexPlan vertexPlan,
    String vertexName, Configuration conf, EventHandler eventHandler,
    TaskAttemptListener taskAttemptListener, Clock clock,
    TaskHeartbeatHandler thh, boolean commitVertexOutputs,
    AppContext appContext, VertexLocationHint vertexLocationHint,
    Map<String, VertexGroupInfo> dagVertexGroups, JavaProfilerOptions profilerOpts) {
  this.vertexId = vertexId;
  this.vertexPlan = vertexPlan;
  this.vertexName = StringInterner.weakIntern(vertexName);
  this.conf = conf;
  this.clock = clock;
  this.appContext = appContext;
  this.commitVertexOutputs = commitVertexOutputs;

  this.taskAttemptListener = taskAttemptListener;
  this.taskHeartbeatHandler = thh;
  this.eventHandler = eventHandler;
  ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
  this.readLock = readWriteLock.readLock();
  this.writeLock = readWriteLock.writeLock();

  if (LOG.isDebugEnabled()) {
    logLocationHints(this.vertexName, vertexLocationHint);
  }
  setTaskLocationHints(vertexLocationHint);
  
  this.dagUgi = appContext.getCurrentDAG().getDagUGI();

  this.taskResource = DagTypeConverters
      .createResourceRequestFromTaskConfig(vertexPlan.getTaskConfig());
  this.processorDescriptor = DagTypeConverters
      .convertProcessorDescriptorFromDAGPlan(vertexPlan
          .getProcessorDescriptor());
  this.localResources = DagTypeConverters
      .createLocalResourceMapFromDAGPlan(vertexPlan.getTaskConfig()
          .getLocalResourceList());
  this.localResources.putAll(appContext.getSessionResources());
  this.environment = DagTypeConverters
      .createEnvironmentMapFromDAGPlan(vertexPlan.getTaskConfig()
          .getEnvironmentSettingList());
  this.javaOpts = vertexPlan.getTaskConfig().hasJavaOpts() ? vertexPlan
      .getTaskConfig().getJavaOpts() : null;
  this.profilerOpts = profilerOpts;
  this.containerContext = new ContainerContext(this.localResources,
      appContext.getCurrentDAG().getCredentials(), this.environment, this.javaOpts, this);

  if (vertexPlan.getInputsCount() > 0) {
    setAdditionalInputs(vertexPlan.getInputsList());
  }
  if (vertexPlan.getOutputsCount() > 0) {
    setAdditionalOutputs(vertexPlan.getOutputsList());
  }

  // Setup the initial parallelism early. This may be changed after
  // initialization or on a setParallelism call.
  this.numTasks = vertexPlan.getTaskConfig().getNumTasks();

  this.dagVertexGroups = dagVertexGroups;

  logIdentifier =  this.getVertexId() + " [" + this.getName() + "]";
  // This "this leak" is okay because the retained pointer is in an
  //  instance variable.
  stateMachine = stateMachineFactory.make(this);
}
 
Example 20
Source File: TaskInfo.java    From tez with Apache License 2.0 4 votes vote down vote up
TaskInfo(JSONObject jsonObject) throws JSONException {
  super(jsonObject);

  Preconditions.checkArgument(
      jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase
          (Constants.TEZ_TASK_ID));

  taskId = StringInterner.weakIntern(jsonObject.optString(Constants.ENTITY));

  //Parse additional Info
  final JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO);

  long sTime = otherInfoNode.optLong(Constants.START_TIME);
  long eTime = otherInfoNode.optLong(Constants.FINISH_TIME);
  if (eTime < sTime) {
    LOG.warn("Task has got wrong start/end values. "
        + "startTime=" + sTime + ", endTime=" + eTime + ". Will check "
        + "timestamps in DAG started/finished events");

    // Check if events TASK_STARTED, TASK_FINISHED can be made use of
    for(Event event : eventList) {
      switch (HistoryEventType.valueOf(event.getType())) {
      case TASK_STARTED:
        sTime = event.getAbsoluteTime();
        break;
      case TASK_FINISHED:
        eTime = event.getAbsoluteTime();
        break;
      default:
        break;
      }
    }

    if (eTime < sTime) {
      LOG.warn("Task has got wrong start/end values in events as well. "
          + "startTime=" + sTime + ", endTime=" + eTime);
    }
  }
  startTime = sTime;
  endTime = eTime;

  diagnostics = otherInfoNode.optString(Constants.DIAGNOSTICS);
  successfulAttemptId = StringInterner.weakIntern(
      otherInfoNode.optString(Constants.SUCCESSFUL_ATTEMPT_ID));
  scheduledTime = otherInfoNode.optLong(Constants.SCHEDULED_TIME);
  status = StringInterner.weakIntern(otherInfoNode.optString(Constants.STATUS));
}