Java Code Examples for org.codehaus.jettison.json.JSONObject#optString()

The following examples show how to use org.codehaus.jettison.json.JSONObject#optString() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: tez   File: Utils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Parse events from json
 *
 * @param eventNodes
 * @param eventList
 * @throws JSONException
 */
public static void parseEvents(JSONArray eventNodes, List<Event> eventList) throws
    JSONException {
  if (eventNodes == null) {
    return;
  }
  for (int i = 0; i < eventNodes.length(); i++) {
    JSONObject eventNode = eventNodes.optJSONObject(i);
    final String eventInfo = eventNode.optString(Constants.EVENT_INFO);
    final String eventType = eventNode.optString(Constants.EVENT_TYPE);
    final long time = eventNode.optLong(Constants.EVENT_TIME_STAMP);

    Event event = new Event(eventInfo, eventType, time);

    eventList.add(event);

  }
}
 
Example 2
Source Project: tez   File: DagInfo.java    License: Apache License 2.0 6 votes vote down vote up
private void parseDAGContext(JSONObject callerContextInfo) {
  if (callerContextInfo == null) {
    LOG.info("No DAG Caller Context available");
    return;
  }
  String context = callerContextInfo.optString(Constants.CONTEXT);
  String callerId = callerContextInfo.optString(Constants.CALLER_ID);
  String callerType = callerContextInfo.optString(Constants.CALLER_TYPE);
  String description = callerContextInfo.optString(Constants.DESCRIPTION);

  this.callerContext = CallerContext.create(context, description);
  if (callerId != null && !callerId.isEmpty() && callerType != null && !callerType.isEmpty()) {
    this.callerContext.setCallerIdAndType(callerId, callerType);
  } else {
    LOG.info("No DAG Caller Context Id and Type available");
  }

}
 
Example 3
Source Project: tez   File: ATSFileParser.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Parse TezApplication json
 *
 * @param tezApplicationJson
 * @throws JSONException
 */
private void processApplication(JSONObject tezApplicationJson) throws JSONException {
  if (tezApplicationJson != null) {
    LOG.debug("Started parsing tez application");
    JSONObject otherInfoNode = tezApplicationJson.optJSONObject(Constants.OTHER_INFO);
    if (otherInfoNode != null) {
      JSONObject tezVersion = otherInfoNode.optJSONObject(Constants.TEZ_VERSION);
      if (tezVersion != null) {
        String version = tezVersion.optString(Constants.VERSION);
        String buildTime = tezVersion.optString(Constants.BUILD_TIME);
        String revision = tezVersion.optString(Constants.REVISION);
        this.versionInfo = new VersionInfo(version, revision, buildTime);
      }
      //Parse Config info
      this.config = Maps.newHashMap();
      JSONObject configNode = otherInfoNode.getJSONObject(Constants.CONFIG);
      Iterator it = configNode.keys();
      while(it.hasNext()) {
        String key = (String) it.next();
        String value = configNode.getString(key);
        config.put(key, value);
      }
    }
    LOG.debug("Finished parsing tez application");
  }
}
 
Example 4
Source Project: tez   File: DAGClientTimelineImpl.java    License: Apache License 2.0 5 votes vote down vote up
private VertexStatusProto.Builder parseVertexStatus(JSONObject jsonRoot,
                                                    Set<StatusGetOpts> statusOptions)
    throws JSONException {
  final JSONObject otherInfoNode = jsonRoot.getJSONObject(ATSConstants.OTHER_INFO);
  final VertexStatusProto.Builder vertexStatusBuilder = VertexStatusProto.newBuilder();

  final String status = otherInfoNode.optString(ATSConstants.STATUS);
  final String diagnostics = otherInfoNode.optString(ATSConstants.DIAGNOSTICS);
  if (status.equals("")) {
    return null;
  }

  vertexStatusBuilder.setState(vertexStateProtoMap.get(status))
      .addAllDiagnostics(Collections.singleton(diagnostics));

  int numRunningTasks = otherInfoNode.optInt(ATSConstants.NUM_TASKS) -
      otherInfoNode.optInt(ATSConstants.NUM_COMPLETED_TASKS);
  ProgressProto.Builder progressBuilder = ProgressProto.newBuilder();
  progressBuilder.setTotalTaskCount(otherInfoNode.optInt(ATSConstants.NUM_TASKS));
  progressBuilder.setRunningTaskCount(numRunningTasks);
  progressBuilder.setSucceededTaskCount(otherInfoNode.optInt(ATSConstants.NUM_SUCCEEDED_TASKS));
  progressBuilder.setKilledTaskCount(otherInfoNode.optInt(ATSConstants.NUM_KILLED_TASKS));
  progressBuilder.setFailedTaskCount(otherInfoNode.optInt(ATSConstants.NUM_FAILED_TASKS));
  vertexStatusBuilder.setProgress(progressBuilder);

  if (statusOptions != null && statusOptions.contains(StatusGetOpts.GET_COUNTERS)) {
    final TezCountersProto.Builder tezCounterBuilder;
    final JSONObject countersNode = otherInfoNode.optJSONObject(ATSConstants.COUNTERS);
    tezCounterBuilder = parseDagCounters(countersNode);
    if (tezCounterBuilder != null) {
      vertexStatusBuilder.setVertexCounters(tezCounterBuilder);
    }
  }

  return vertexStatusBuilder;
}
 
Example 5
Source Project: tez   File: DAGClientTimelineImpl.java    License: Apache License 2.0 5 votes vote down vote up
private TezCounterGroupProto.Builder parseCounterGroup(JSONObject counterGroupNode)
    throws JSONException {

  if (counterGroupNode == null) {
    return null;
  }

  TezCounterGroupProto.Builder counterGroup = TezCounterGroupProto.newBuilder();

  final String groupName = counterGroupNode.optString(ATSConstants.COUNTER_GROUP_NAME);
  final String groupDisplayName = counterGroupNode.optString(
      ATSConstants.COUNTER_GROUP_DISPLAY_NAME, groupName);
  final JSONArray counterNodes = counterGroupNode.optJSONArray(ATSConstants.COUNTERS);
  final int numCounters = counterNodes.length();

  List<TezCounterProto> counters = new ArrayList<TezCounterProto>(numCounters);

  for (int i = 0; i < numCounters; i++) {
    final JSONObject counterNode = counterNodes.getJSONObject(i);
    final String counterName = counterNode.getString(ATSConstants.COUNTER_NAME);
    final String counterDisplayName = counterNode.optString(ATSConstants.COUNTER_DISPLAY_NAME,
        counterName);
    final long counterValue = counterNode.getLong(ATSConstants.COUNTER_VALUE);

    counters.add(
        TezCounterProto.newBuilder()
            .setName(counterName)
            .setDisplayName(counterDisplayName)
            .setValue(counterValue)
            .build());
  }

  return counterGroup.setName(groupName)
      .setDisplayName(groupDisplayName)
      .addAllCounters(counters);
}
 
Example 6
Source Project: tez   File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Parse tez counters from json
 *
 * @param jsonObject
 * @return TezCounters
 * @throws JSONException
 */
public static TezCounters parseTezCountersFromJSON(JSONObject jsonObject)
    throws JSONException {
  TezCounters counters = new TezCounters();

  if (jsonObject == null) {
    return counters; //empty counters.
  }

  final JSONArray counterGroupNodes = jsonObject.optJSONArray(Constants.COUNTER_GROUPS);
  if (counterGroupNodes != null) {
    for (int i = 0; i < counterGroupNodes.length(); i++) {
      JSONObject counterGroupNode = counterGroupNodes.optJSONObject(i);
      final String groupName = counterGroupNode.optString(Constants.COUNTER_GROUP_NAME);
      final String groupDisplayName = counterGroupNode.optString(
          Constants.COUNTER_GROUP_DISPLAY_NAME, groupName);

      CounterGroup group = counters.addGroup(groupName, groupDisplayName);

      final JSONArray counterNodes = counterGroupNode.optJSONArray(Constants.COUNTERS);

      //Parse counter nodes
      for (int j = 0; j < counterNodes.length(); j++) {
        JSONObject counterNode = counterNodes.optJSONObject(j);
        final String counterName = counterNode.getString(Constants.COUNTER_NAME);
        final String counterDisplayName =
            counterNode.optString(Constants.COUNTER_DISPLAY_NAME, counterName);
        final long counterValue = counterNode.getLong(Constants.COUNTER_VALUE);
        addCounter(group, counterName, counterDisplayName, counterValue);
      }
    }
  }
  return counters;
}
 
Example 7
Source Project: tez   File: DagInfo.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Parse edge details in the DAG
 *
 * @param edgesArray
 *
 * @throws JSONException
 */
private void parseEdges(JSONArray edgesArray) throws JSONException {
  if (edgesArray == null) {
    return;
  }
  for (int i = 0; i < edgesArray.length(); i++) {
    JSONObject edge = edgesArray.getJSONObject(i);
    Integer edgeId = edge.optInt(Constants.EDGE_ID);
    String inputVertexName =
        edge.optString(Constants.INPUT_VERTEX_NAME);
    String outputVertexName =
        edge.optString(Constants.OUTPUT_VERTEX_NAME);
    String dataMovementType =
        edge.optString(Constants.DATA_MOVEMENT_TYPE);
    String edgeSourceClass =
        edge.optString(Constants.EDGE_SOURCE_CLASS);
    String edgeDestinationClass =
        edge.optString(Constants.EDGE_DESTINATION_CLASS);
    String inputUserPayloadAsText =
        edge.optString(Constants.INPUT_PAYLOAD_TEXT);
    String outputUserPayloadAsText =
        edge.optString(Constants.OUTPUT_PAYLOAD_TEXT);
    EdgeInfo edgeInfo = new EdgeInfo(inputVertexName, outputVertexName,
        dataMovementType, edgeSourceClass, edgeDestinationClass, inputUserPayloadAsText,
        outputUserPayloadAsText);
    edgeInfoMap.put(edgeId, edgeInfo);
  }
}
 
Example 8
Source Project: Bats   File: LogicalPlanSerializer.java    License: Apache License 2.0 4 votes vote down vote up
public static PropertiesConfiguration convertToProperties(JSONObject json) throws JSONException
{
  PropertiesConfiguration props = new PropertiesConfiguration();
  JSONArray allOperators = json.getJSONArray("operators");
  JSONArray allStreams = json.getJSONArray("streams");

  for (int j = 0; j < allOperators.length(); j++) {
    JSONObject operatorDetail = allOperators.getJSONObject(j);
    String operatorName = operatorDetail.getString("name");
    String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorName;
    props.setProperty(operatorKey + ".classname", operatorDetail.getString("class"));
    JSONObject properties = operatorDetail.optJSONObject("properties");
    if (properties != null) {
      Iterator<String> iter2 = properties.keys();
      while (iter2.hasNext()) {
        String propertyName = iter2.next();
        if (!propertyName.equals("name") && !propertyName.equals("class") && properties.opt(propertyName) != null) {
          JSONArray list = properties.optJSONArray(propertyName);
          String value = "";
          if (list != null) {
            for (int i = 0; i < list.length(); i++) {
              if (i != 0) {
                value += ",";
              }
              value += list.get(i).toString();
            }
            props.setProperty(operatorKey + "." + propertyName, value);
          } else {
            props.setProperty(operatorKey + "." + propertyName, properties.get(propertyName));
          }
        }
      }
    }
  }

  for (int j = 0; j < allStreams.length(); j++) {
    JSONObject streamDetail = allStreams.getJSONObject(j);
    String streamName = streamDetail.getString("name");
    String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamName;
    JSONObject sourceDetail = streamDetail.getJSONObject("source");
    JSONArray sinksList = streamDetail.getJSONArray("sinks");

    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, sourceDetail.getString("operatorName") + "." + sourceDetail.getString("portName"));
    String sinksValue = "";
    for (int i = 0; i < sinksList.length(); i++) {
      if (!sinksValue.isEmpty()) {
        sinksValue += ",";
      }
      sinksValue += sinksList.getJSONObject(i).getString("operatorName") + "." + sinksList.getJSONObject(i).getString("portName");
    }
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
    String locality = streamDetail.optString("locality", null);
    if (locality != null) {
      props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, Locality.valueOf(locality));
    }
  }

  // TBD: Attributes

  return props;
}
 
Example 9
public static PropertiesConfiguration convertToProperties(JSONObject json) throws JSONException
{
  PropertiesConfiguration props = new PropertiesConfiguration();
  JSONArray allOperators = json.getJSONArray("operators");
  JSONArray allStreams = json.getJSONArray("streams");

  for (int j = 0; j < allOperators.length(); j++) {
    JSONObject operatorDetail = allOperators.getJSONObject(j);
    String operatorName = operatorDetail.getString("name");
    String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorName;
    props.setProperty(operatorKey + ".classname", operatorDetail.getString("class"));
    JSONObject properties = operatorDetail.optJSONObject("properties");
    if (properties != null) {
      Iterator<String> iter2 = properties.keys();
      while (iter2.hasNext()) {
        String propertyName = iter2.next();
        if (!propertyName.equals("name") && !propertyName.equals("class") && properties.opt(propertyName) != null) {
          JSONArray list = properties.optJSONArray(propertyName);
          String value = "";
          if (list != null) {
            for (int i = 0; i < list.length(); i++) {
              if (i != 0) {
                value += ",";
              }
              value += list.get(i).toString();
            }
            props.setProperty(operatorKey + "." + propertyName, value);
          } else {
            props.setProperty(operatorKey + "." + propertyName, properties.get(propertyName));
          }
        }
      }
    }
  }

  for (int j = 0; j < allStreams.length(); j++) {
    JSONObject streamDetail = allStreams.getJSONObject(j);
    String streamName = streamDetail.getString("name");
    String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamName;
    JSONObject sourceDetail = streamDetail.getJSONObject("source");
    JSONArray sinksList = streamDetail.getJSONArray("sinks");

    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, sourceDetail.getString("operatorName") + "." + sourceDetail.getString("portName"));
    String sinksValue = "";
    for (int i = 0; i < sinksList.length(); i++) {
      if (!sinksValue.isEmpty()) {
        sinksValue += ",";
      }
      sinksValue += sinksList.getJSONObject(i).getString("operatorName") + "." + sinksList.getJSONObject(i).getString("portName");
    }
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
    String locality = streamDetail.optString("locality", null);
    if (locality != null) {
      props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, Locality.valueOf(locality));
    }
  }

  // TBD: Attributes

  return props;
}
 
Example 10
Source Project: tez   File: DAGClientTimelineImpl.java    License: Apache License 2.0 4 votes vote down vote up
private DAGStatusProto.Builder parseDagStatus(JSONObject jsonRoot, Set<StatusGetOpts> statusOptions)
    throws JSONException, TezException {
  final JSONObject otherInfoNode = jsonRoot.getJSONObject(ATSConstants.OTHER_INFO);

  DAGStatusProto.Builder dagStatusBuilder = DAGStatusProto.newBuilder();

  final String status = otherInfoNode.optString(ATSConstants.STATUS);
  final String diagnostics = otherInfoNode.optString(ATSConstants.DIAGNOSTICS);
  if (status.equals("")) {
    return null;
  }

  dagStatusBuilder.setState(dagStateProtoMap.get(status))
      .addAllDiagnostics(Collections.singleton(diagnostics));

  if (statusOptions != null && statusOptions.contains(StatusGetOpts.GET_COUNTERS)) {
    final TezCountersProto.Builder tezCounterBuilder;
    final JSONObject countersNode = otherInfoNode.optJSONObject(ATSConstants.COUNTERS);
    tezCounterBuilder = parseDagCounters(countersNode);
    if (tezCounterBuilder != null) {
      dagStatusBuilder.setDagCounters(tezCounterBuilder);
    }
  }

  final Map<String, VertexTaskStats> vertexTaskStatsMap = parseTaskStatsForVertexes();
  if (vertexTaskStatsMap.size() > 0) {
    ProgressProto.Builder dagProgressBuilder = getProgressBuilder(vertexTaskStatsMap, null);
    dagStatusBuilder.setDAGProgress(dagProgressBuilder);

    List<StringProgressPairProto> vertexProgressBuilder =
        new ArrayList<StringProgressPairProto>(vertexTaskStatsMap.size());
    for (Map.Entry<String, VertexTaskStats> v : vertexTaskStatsMap.entrySet()) {
      StringProgressPairProto vertexProgressProto = StringProgressPairProto
          .newBuilder()
          .setKey(v.getKey())
          .setProgress(getProgressBuilder(vertexTaskStatsMap, v.getKey()))
          .build();
      vertexProgressBuilder.add(vertexProgressProto);
    }
    dagStatusBuilder.addAllVertexProgress(vertexProgressBuilder);
  }

  return dagStatusBuilder;
}
 
Example 11
Source Project: tez   File: SimpleHistoryParser.java    License: Apache License 2.0 4 votes vote down vote up
protected void readEventsFromSource(String dagId, JSONObjectSource source,
    Map<String, JSONObject> vertexJsonMap, Map<String, JSONObject> taskJsonMap,
    Map<String, JSONObject> attemptJsonMap) throws JSONException, TezException, IOException{
  JSONObject dagJson = null;
  TezDAGID tezDAGID = TezDAGID.fromString(dagId);
  String userName = null;

  while (source.hasNext()) {
    JSONObject jsonObject = source.next();

    String entity = jsonObject.getString(Constants.ENTITY);
    String entityType = jsonObject.getString(Constants.ENTITY_TYPE);
    switch (entityType) {
    case Constants.TEZ_DAG_ID:
      if (!dagId.equals(entity)) {
        LOG.warn(dagId + " is not matching with " + entity);
        continue;
      }
      // Club all DAG related information together (DAG_INIT, DAG_FINISH etc). Each of them
      // would have a set of entities in otherinfo (e.g vertex mapping, dagPlan, start/finish
      // time etc).
      if (dagJson == null) {
        dagJson = jsonObject;
      } else if (dagJson.optJSONObject(ATSConstants.OTHER_INFO)
          .optJSONObject(ATSConstants.DAG_PLAN) == null) {
        // if DAG_PLAN is not filled already, let's try to fetch it from other
        dagJson.getJSONObject(ATSConstants.OTHER_INFO).put(ATSConstants.DAG_PLAN, jsonObject
            .getJSONObject(ATSConstants.OTHER_INFO).getJSONObject(ATSConstants.DAG_PLAN));
      }
      JSONArray relatedEntities = dagJson.optJSONArray(Constants
          .RELATED_ENTITIES);
      //UserName is present in related entities
      // {"entity":"userXYZ","entitytype":"user"}
      if (relatedEntities != null) {
        for (int i = 0; i < relatedEntities.length(); i++) {
          JSONObject subEntity = relatedEntities.getJSONObject(i);
          String subEntityType = subEntity.optString(Constants.ENTITY_TYPE);
          if (subEntityType != null && subEntityType.equals(Constants.USER)) {
            userName = subEntity.getString(Constants.ENTITY);
            break;
          }
        }
      }
      populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO),
          dagJson.getJSONObject(Constants.OTHER_INFO));
      break;
    case Constants.TEZ_VERTEX_ID:
      String vertexName = entity;
      TezVertexID tezVertexID = TezVertexID.fromString(vertexName);
      if (!tezDAGID.equals(tezVertexID.getDAGId())) {
        LOG.warn("{} does not belong to {} ('{}' != '{}')}", vertexName, tezDAGID, tezDAGID, tezVertexID.getDAGId());
        continue;
      }
      if (!vertexJsonMap.containsKey(vertexName)) {
        vertexJsonMap.put(vertexName, jsonObject);
      }
      populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO), vertexName, vertexJsonMap);
      break;
    case Constants.TEZ_TASK_ID:
      String taskName = entity;
      TezTaskID tezTaskID = TezTaskID.fromString(taskName);
      if (!tezDAGID.equals(tezTaskID.getVertexID().getDAGId())) {
        LOG.warn("{} does not belong to {} ('{}' != '{}')}", taskName, tezDAGID, tezDAGID,
            tezTaskID.getVertexID().getDAGId());
        continue;
      }
      if (!taskJsonMap.containsKey(taskName)) {
        taskJsonMap.put(taskName, jsonObject);
      }
      populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO), taskName, taskJsonMap);
      break;
    case Constants.TEZ_TASK_ATTEMPT_ID:
      String taskAttemptName = entity;
      TezTaskAttemptID tezAttemptId = TezTaskAttemptID.fromString(taskAttemptName);
      if (!tezDAGID.equals(tezAttemptId.getTaskID().getVertexID().getDAGId())) {
        LOG.warn("{} does not belong to {} ('{}' != '{}')}", taskAttemptName, tezDAGID, tezDAGID,
            tezAttemptId.getTaskID().getVertexID().getDAGId());
        continue;
      }
      if (!attemptJsonMap.containsKey(taskAttemptName)) {
        attemptJsonMap.put(taskAttemptName, jsonObject);
      }
      populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO), taskAttemptName, attemptJsonMap);
      break;
    default:
      break;
    }
  }
  source.close();
  if (dagJson != null) {
    this.dagInfo = DagInfo.create(dagJson);
    setUserName(userName);
  } else {
    LOG.error("Dag is not yet parsed. Looks like partial file.");
    throw new TezException(
        "Please provide a valid/complete history log file containing " + dagId);
  }
}
 
Example 12
Source Project: tez   File: TaskAttemptInfo.java    License: Apache License 2.0 4 votes vote down vote up
TaskAttemptInfo(JSONObject jsonObject) throws JSONException {
  super(jsonObject);

  Preconditions.checkArgument(
      jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase
          (Constants.TEZ_TASK_ATTEMPT_ID));

  taskAttemptId = StringInterner.weakIntern(jsonObject.optString(Constants.ENTITY));

  //Parse additional Info
  final JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO);

  long sTime = otherInfoNode.optLong(Constants.START_TIME);
  long eTime = otherInfoNode.optLong(Constants.FINISH_TIME);
  if (eTime < sTime) {
    LOG.warn("TaskAttemptInfo has got wrong start/end values. "
        + "startTime=" + sTime + ", endTime=" + eTime + ". Will check "
        + "timestamps in DAG started/finished events");

    // Check if events TASK_STARTED, TASK_FINISHED can be made use of
    for(Event event : eventList) {
      switch (HistoryEventType.valueOf(event.getType())) {
      case TASK_ATTEMPT_STARTED:
        sTime = event.getAbsoluteTime();
        break;
      case TASK_ATTEMPT_FINISHED:
        eTime = event.getAbsoluteTime();
        break;
      default:
        break;
      }
    }

    if (eTime < sTime) {
      LOG.warn("TaskAttemptInfo has got wrong start/end values in events as well. "
          + "startTime=" + sTime + ", endTime=" + eTime);
    }
  }
  startTime = sTime;
  endTime = eTime;

  diagnostics = otherInfoNode.optString(Constants.DIAGNOSTICS);
  creationTime = otherInfoNode.optLong(Constants.CREATION_TIME);
  creationCausalTA = StringInterner.weakIntern(
      otherInfoNode.optString(Constants.CREATION_CAUSAL_ATTEMPT));
  allocationTime = otherInfoNode.optLong(Constants.ALLOCATION_TIME);
  containerId = StringInterner.weakIntern(otherInfoNode.optString(Constants.CONTAINER_ID));
  String id = otherInfoNode.optString(Constants.NODE_ID);
  nodeId = StringInterner.weakIntern((id != null) ? (id.split(":")[0]) : "");
  logUrl = otherInfoNode.optString(Constants.COMPLETED_LOGS_URL);

  status = StringInterner.weakIntern(otherInfoNode.optString(Constants.STATUS));
  container = new Container(containerId, nodeId);
  if (otherInfoNode.has(Constants.LAST_DATA_EVENTS)) {
    List<DataDependencyEvent> eventInfo = Utils.parseDataEventDependencyFromJSON(
        otherInfoNode.optJSONObject(Constants.LAST_DATA_EVENTS));
    long lastTime = 0;
    for (DataDependencyEvent item : eventInfo) {
      // check these are in time order
      Preconditions.checkState(lastTime < item.getTimestamp());
      lastTime = item.getTimestamp();
      lastDataEvents.add(item);
    }
  }
  terminationCause = StringInterner
      .weakIntern(otherInfoNode.optString(ATSConstants.TASK_ATTEMPT_ERROR_ENUM));
  executionTimeInterval = (endTime > startTime) ? (endTime - startTime) : 0;
}
 
Example 13
Source Project: tez   File: DagInfo.java    License: Apache License 2.0 4 votes vote down vote up
DagInfo(JSONObject jsonObject) throws JSONException {
  super(jsonObject);

  vertexNameMap = Maps.newHashMap();
  vertexNameIDMapping = new DualHashBidiMap<>();
  edgeInfoMap = Maps.newHashMap();
  basicVertexInfoMap = Maps.newHashMap();
  containerMapping = LinkedHashMultimap.create();

  Preconditions.checkArgument(jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase
      (Constants.TEZ_DAG_ID));

  dagId = StringInterner.weakIntern(jsonObject.getString(Constants.ENTITY));

  //Parse additional Info
  JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO);

  long sTime = otherInfoNode.optLong(Constants.START_TIME);
  long eTime= otherInfoNode.optLong(Constants.FINISH_TIME);
  userName = otherInfoNode.optString(Constants.USER);
  if (eTime < sTime) {
    LOG.warn("DAG has got wrong start/end values. "
        + "startTime=" + sTime + ", endTime=" + eTime + ". Will check "
        + "timestamps in DAG started/finished events");

    // Check if events DAG_STARTED, DAG_FINISHED can be made use of
    for(Event event : eventList) {
      switch (HistoryEventType.valueOf(event.getType())) {
      case DAG_STARTED:
        sTime = event.getAbsoluteTime();
        break;
      case DAG_FINISHED:
        eTime = event.getAbsoluteTime();
        break;
      default:
        break;
      }
    }

    if (eTime < sTime) {
      LOG.warn("DAG has got wrong start/end values in events as well. "
          + "startTime=" + sTime + ", endTime=" + eTime);
    }
  }
  startTime = sTime;
  endTime = eTime;

  //TODO: Not getting populated correctly for lots of jobs.  Verify
  submitTime = otherInfoNode.optLong(Constants.START_REQUESTED_TIME);
  diagnostics = otherInfoNode.optString(Constants.DIAGNOSTICS);
  failedTasks = otherInfoNode.optInt(Constants.NUM_FAILED_TASKS);
  JSONObject dagPlan = otherInfoNode.optJSONObject(Constants.DAG_PLAN);
  name = StringInterner.weakIntern((dagPlan != null) ? (dagPlan.optString(Constants.DAG_NAME)) : null);
  if (dagPlan != null) {
    JSONArray vertices = dagPlan.optJSONArray(Constants.VERTICES);
    if (vertices != null) {
      numVertices = vertices.length();
    } else {
      numVertices = 0;
    }
    parseDAGPlan(dagPlan);
  } else {
    numVertices = 0;
  }
  status = StringInterner.weakIntern(otherInfoNode.optString(Constants.STATUS));

  //parse name id mapping
  JSONObject vertexIDMappingJson = otherInfoNode.optJSONObject(Constants.VERTEX_NAME_ID_MAPPING);
  if (vertexIDMappingJson != null) {
    //get vertex name
    for (Map.Entry<String, BasicVertexInfo> entry : basicVertexInfoMap.entrySet()) {
      String vertexId = vertexIDMappingJson.optString(entry.getKey());
      //vertexName --> vertexId
      vertexNameIDMapping.put(entry.getKey(), vertexId);
    }
  }
}
 
Example 14
Source Project: tez   File: DagInfo.java    License: Apache License 2.0 4 votes vote down vote up
private void parseBasicVertexInfo(JSONArray verticesInfo) throws JSONException {
  if (verticesInfo == null) {
    LOG.info("No vertices available.");
    return;
  }

  //Parse basic information available in DAG for vertex and edges
  for (int i = 0; i < verticesInfo.length(); i++) {
    BasicVertexInfo basicVertexInfo = new BasicVertexInfo();

    JSONObject vJson = verticesInfo.getJSONObject(i);
    basicVertexInfo.vertexName =
        vJson.optString(Constants.VERTEX_NAME);
    JSONArray inEdges = vJson.optJSONArray(Constants.IN_EDGE_IDS);
    if (inEdges != null) {
      String[] inEdgeIds = new String[inEdges.length()];
      for (int j = 0; j < inEdges.length(); j++) {
        inEdgeIds[j] = inEdges.get(j).toString();
      }
      basicVertexInfo.inEdgeIds = inEdgeIds;
    }

    JSONArray outEdges = vJson.optJSONArray(Constants.OUT_EDGE_IDS);
    if (outEdges != null) {
      String[] outEdgeIds = new String[outEdges.length()];
      for (int j = 0; j < outEdges.length(); j++) {
        outEdgeIds[j] = outEdges.get(j).toString();
      }
      basicVertexInfo.outEdgeIds = outEdgeIds;
    }

    JSONArray addInputsJson =
        vJson.optJSONArray(Constants.ADDITIONAL_INPUTS);
    basicVertexInfo.additionalInputs = parseAdditionalDetailsForVertex(addInputsJson);

    JSONArray addOutputsJson =
        vJson.optJSONArray(Constants.ADDITIONAL_OUTPUTS);
    basicVertexInfo.additionalOutputs = parseAdditionalDetailsForVertex(addOutputsJson);

    basicVertexInfoMap.put(basicVertexInfo.vertexName, basicVertexInfo);
  }
}
 
Example 15
Source Project: tez   File: TaskInfo.java    License: Apache License 2.0 4 votes vote down vote up
TaskInfo(JSONObject jsonObject) throws JSONException {
  super(jsonObject);

  Preconditions.checkArgument(
      jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase
          (Constants.TEZ_TASK_ID));

  taskId = StringInterner.weakIntern(jsonObject.optString(Constants.ENTITY));

  //Parse additional Info
  final JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO);

  long sTime = otherInfoNode.optLong(Constants.START_TIME);
  long eTime = otherInfoNode.optLong(Constants.FINISH_TIME);
  if (eTime < sTime) {
    LOG.warn("Task has got wrong start/end values. "
        + "startTime=" + sTime + ", endTime=" + eTime + ". Will check "
        + "timestamps in DAG started/finished events");

    // Check if events TASK_STARTED, TASK_FINISHED can be made use of
    for(Event event : eventList) {
      switch (HistoryEventType.valueOf(event.getType())) {
      case TASK_STARTED:
        sTime = event.getAbsoluteTime();
        break;
      case TASK_FINISHED:
        eTime = event.getAbsoluteTime();
        break;
      default:
        break;
      }
    }

    if (eTime < sTime) {
      LOG.warn("Task has got wrong start/end values in events as well. "
          + "startTime=" + sTime + ", endTime=" + eTime);
    }
  }
  startTime = sTime;
  endTime = eTime;

  diagnostics = otherInfoNode.optString(Constants.DIAGNOSTICS);
  successfulAttemptId = StringInterner.weakIntern(
      otherInfoNode.optString(Constants.SUCCESSFUL_ATTEMPT_ID));
  scheduledTime = otherInfoNode.optLong(Constants.SCHEDULED_TIME);
  status = StringInterner.weakIntern(otherInfoNode.optString(Constants.STATUS));
}
 
Example 16
Source Project: tez   File: VertexInfo.java    License: Apache License 2.0 4 votes vote down vote up
VertexInfo(JSONObject jsonObject) throws JSONException {
  super(jsonObject);

  Preconditions.checkArgument(
      jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase
          (Constants.TEZ_VERTEX_ID));

  vertexId = StringInterner.weakIntern(jsonObject.optString(Constants.ENTITY));
  taskInfoMap = Maps.newHashMap();

  inEdgeList = Lists.newLinkedList();
  outEdgeList = Lists.newLinkedList();
  additionalInputInfoList = Lists.newLinkedList();
  additionalOutputInfoList = Lists.newLinkedList();

  //Parse additional Info
  JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO);
  initRequestedTime = otherInfoNode.optLong(Constants.INIT_REQUESTED_TIME);
  startRequestedTime = otherInfoNode.optLong(Constants.START_REQUESTED_TIME);

  long sTime = otherInfoNode.optLong(Constants.START_TIME);
  long iTime = otherInfoNode.optLong(Constants.INIT_TIME);
  long eTime = otherInfoNode.optLong(Constants.FINISH_TIME);
  if (eTime < sTime) {
    LOG.warn("Vertex has got wrong start/end values. "
        + "startTime=" + sTime + ", endTime=" + eTime + ". Will check "
        + "timestamps in DAG started/finished events");

    // Check if events VERTEX_STARTED, VERTEX_FINISHED can be made use of
    for(Event event : eventList) {
      switch (HistoryEventType.valueOf(event.getType())) {
      case VERTEX_INITIALIZED:
        iTime = event.getAbsoluteTime();
        break;
      case VERTEX_STARTED:
        sTime = event.getAbsoluteTime();
        break;
      case VERTEX_FINISHED:
        eTime = event.getAbsoluteTime();
        break;
      default:
        break;
      }
    }

    if (eTime < sTime) {
      LOG.warn("Vertex has got wrong start/end values in events as well. "
          + "startTime=" + sTime + ", endTime=" + eTime);
    }
  }
  startTime = sTime;
  finishTime = eTime;
  initTime = iTime;


  diagnostics = otherInfoNode.optString(Constants.DIAGNOSTICS);
  numTasks = otherInfoNode.optInt(Constants.NUM_TASKS);
  failedTasks = otherInfoNode.optInt(Constants.NUM_FAILED_TASKS);
  succeededTasks =
      otherInfoNode.optInt(Constants.NUM_SUCCEEDED_TASKS);
  completedTasks =
      otherInfoNode.optInt(Constants.NUM_COMPLETED_TASKS);
  killedTasks = otherInfoNode.optInt(Constants.NUM_KILLED_TASKS);
  numFailedTaskAttempts =
      otherInfoNode.optInt(Constants.NUM_FAILED_TASKS_ATTEMPTS);
  vertexName = StringInterner.weakIntern(otherInfoNode.optString(Constants.VERTEX_NAME));
  processorClass = StringInterner.weakIntern(otherInfoNode.optString(Constants.PROCESSOR_CLASS_NAME));
  status = StringInterner.weakIntern(otherInfoNode.optString(Constants.STATUS));
}