Java Code Examples for org.codehaus.jettison.json.JSONObject#optJSONArray()

The following examples show how to use org.codehaus.jettison.json.JSONObject#optJSONArray() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Bats   File: TypeGraph.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * A utility method that tells whether a class is considered a bean.<br/>
 * For simplicity we exclude classes that have any type-args.
 *
 * @param className name of the class
 * @return true if it is a bean false otherwise.
 */
public boolean isInstantiableBean(String className) throws JSONException
{
  JSONObject classDesc = describeClass(className);
  if (classDesc.has("typeArgs")) {
    //any type with generics is not considered a bean
    return false;
  }
  JSONArray classProps = classDesc.optJSONArray("properties");
  if (classProps == null || classProps.length() == 0) {
    //no properties then cannot be a bean
    return false;
  }
  for (int p = 0; p < classProps.length(); p++) {
    JSONObject propDesc = classProps.getJSONObject(p);
    if (propDesc.optBoolean("canGet", false)) {
      return true;
    }
  }
  return false;
}
 
Example 2
Source Project: appinventor-extensions   File: Compiler.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Processes the conditional info from simple_components_build_info.json into
 * a structure mapping annotation types to component names to block names to
 * values.
 *
 * @param compJson Parsed component data from JSON
 * @param type The name of the type being processed
 * @param targetInfo Name of the annotation target being processed (e.g.,
 *                   permissions). Any of: PERMISSIONS_TARGET,
 *                   BROADCAST_RECEIVERS_TARGET
 */
private void processConditionalInfo(JSONObject compJson, String type, String targetInfo) {
  // Strip off the package name since SCM and BKY use unqualified names
  type = type.substring(type.lastIndexOf('.') + 1);

  JSONObject conditionals = compJson.optJSONObject(ComponentDescriptorConstants.CONDITIONALS_TARGET);
  if (conditionals != null) {
    JSONObject jsonBlockMap = conditionals.optJSONObject(targetInfo);
    if (jsonBlockMap != null) {
      if (!this.conditionals.containsKey(targetInfo)) {
        this.conditionals.put(targetInfo, new HashMap<String, Map<String, Set<String>>>());
      }
      Map<String, Set<String>> blockMap = new HashMap<>();
      this.conditionals.get(targetInfo).put(type, blockMap);
      for (String key : (List<String>) Lists.newArrayList(jsonBlockMap.keys())) {
        JSONArray data = jsonBlockMap.optJSONArray(key);
        HashSet<String> result = new HashSet<>();
        for (int i = 0; i < data.length(); i++) {
          result.add(data.optString(i));
        }
        blockMap.put(key, result);
      }
    }
  }
}
 
Example 3
Source Project: attic-apex-core   File: TypeGraph.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * A utility method that tells whether a class is considered a bean.<br/>
 * For simplicity we exclude classes that have any type-args.
 *
 * @param className name of the class
 * @return true if it is a bean false otherwise.
 */
public boolean isInstantiableBean(String className) throws JSONException
{
  JSONObject classDesc = describeClass(className);
  if (classDesc.has("typeArgs")) {
    //any type with generics is not considered a bean
    return false;
  }
  JSONArray classProps = classDesc.optJSONArray("properties");
  if (classProps == null || classProps.length() == 0) {
    //no properties then cannot be a bean
    return false;
  }
  for (int p = 0; p < classProps.length(); p++) {
    JSONObject propDesc = classProps.getJSONObject(p);
    if (propDesc.optBoolean("canGet", false)) {
      return true;
    }
  }
  return false;
}
 
Example 4
@Test
public void testAdditionalPortInfo() throws Exception
{
  String[] classFilePath = getClassFileInClasspath();
  OperatorDiscoverer operatorDiscoverer = new OperatorDiscoverer(classFilePath);
  operatorDiscoverer.buildTypeGraph();
  JSONObject operator = operatorDiscoverer.describeOperator(SubSubClassGeneric.class.getName());

  JSONObject portClassHierarchy = new JSONObject();
  JSONObject portsWithSchemaClasses = new JSONObject();
  operatorDiscoverer.buildAdditionalPortInfo(operator, portClassHierarchy, portsWithSchemaClasses);

  JSONArray stringTypeArray = portClassHierarchy.optJSONArray("java.lang.String");
  Assert.assertNotNull("string hierarchy", stringTypeArray);

  Assert.assertEquals("number of immediate ancestors", 4, stringTypeArray.length());

  Assert.assertEquals("number of port types with schema", 0, portsWithSchemaClasses.length());
}
 
Example 5
Source Project: tez   File: DAGClientTimelineImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public VertexStatus getVertexStatus(String vertexName, Set<StatusGetOpts> statusOptions)
    throws IOException, TezException {
  final String url = String.format(
      "%s/%s?primaryFilter=%s:%s&secondaryFilter=vertexName:%s&fields=%s", baseUri,
      ATSConstants.TEZ_VERTEX_ID, ATSConstants.TEZ_DAG_ID, dagId, vertexName, FILTER_BY_FIELDS);

  try {
    VertexStatusProto.Builder statusBuilder;
    final JSONObject jsonRoot = getJsonRootEntity(url);
    JSONArray entitiesNode = jsonRoot.optJSONArray(ATSConstants.ENTITIES);
    if (entitiesNode == null || entitiesNode.length() != 1) {
      throw new TezException("Failed to get vertex status YARN Timeline");
    }
    JSONObject vertexNode = entitiesNode.getJSONObject(0);

    statusBuilder = parseVertexStatus(vertexNode, statusOptions);
    if (statusBuilder == null) {
      throw new TezException("Failed to parse vertex status from YARN Timeline");
    }

    return new VertexStatus(statusBuilder);
  } catch (JSONException je) {
    throw new TezException("Failed to parse VertexStatus json from YARN Timeline", je);
  }
}
 
Example 6
Source Project: tez   File: DAGClientTimelineImpl.java    License: Apache License 2.0 6 votes vote down vote up
private TezCountersProto.Builder parseDagCounters(JSONObject countersNode)
    throws JSONException {
  if (countersNode == null) {
    return null;
  }

  TezCountersProto.Builder countersProto = TezCountersProto.newBuilder();
  final JSONArray counterGroupNodes = countersNode.optJSONArray(ATSConstants.COUNTER_GROUPS);
  if (counterGroupNodes != null) {
    final int numCounterGroups = counterGroupNodes.length();

    for (int i = 0; i < numCounterGroups; i++) {
      TezCounterGroupProto.Builder counterGroupBuilder =
          parseCounterGroup(counterGroupNodes.optJSONObject(i));
      if (counterGroupBuilder != null) {
        countersProto.addCounterGroups(counterGroupBuilder);
      }
    }
  }

  return countersProto;
}
 
Example 7
Source Project: tez   File: DAGClientTimelineImpl.java    License: Apache License 2.0 5 votes vote down vote up
private TezCounterGroupProto.Builder parseCounterGroup(JSONObject counterGroupNode)
    throws JSONException {

  if (counterGroupNode == null) {
    return null;
  }

  TezCounterGroupProto.Builder counterGroup = TezCounterGroupProto.newBuilder();

  final String groupName = counterGroupNode.optString(ATSConstants.COUNTER_GROUP_NAME);
  final String groupDisplayName = counterGroupNode.optString(
      ATSConstants.COUNTER_GROUP_DISPLAY_NAME, groupName);
  final JSONArray counterNodes = counterGroupNode.optJSONArray(ATSConstants.COUNTERS);
  final int numCounters = counterNodes.length();

  List<TezCounterProto> counters = new ArrayList<TezCounterProto>(numCounters);

  for (int i = 0; i < numCounters; i++) {
    final JSONObject counterNode = counterNodes.getJSONObject(i);
    final String counterName = counterNode.getString(ATSConstants.COUNTER_NAME);
    final String counterDisplayName = counterNode.optString(ATSConstants.COUNTER_DISPLAY_NAME,
        counterName);
    final long counterValue = counterNode.getLong(ATSConstants.COUNTER_VALUE);

    counters.add(
        TezCounterProto.newBuilder()
            .setName(counterName)
            .setDisplayName(counterDisplayName)
            .setValue(counterValue)
            .build());
  }

  return counterGroup.setName(groupName)
      .setDisplayName(groupDisplayName)
      .addAllCounters(counters);
}
 
Example 8
Source Project: tez   File: DAGClientTimelineImpl.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
protected Map<String, VertexTaskStats> parseTaskStatsForVertexes()
    throws TezException, JSONException {

  if (vertexTaskStatsCache == null) {
    final String url = String.format("%s/%s?primaryFilter=%s:%s&fields=%s", baseUri,
        ATSConstants.TEZ_VERTEX_ID, ATSConstants.TEZ_DAG_ID, dagId, FILTER_BY_FIELDS);

    final JSONObject jsonRoot = getJsonRootEntity(url);
    final JSONArray vertexNodes = jsonRoot.optJSONArray(ATSConstants.ENTITIES);

    if (vertexNodes != null) {
      final int numVertexNodes = vertexNodes.length();
      Map<String, VertexTaskStats> vertexTaskStatsMap =
          new HashMap<String, VertexTaskStats>(numVertexNodes);
      for (int i = 0; i < numVertexNodes; i++) {
        final JSONObject vertexNode = vertexNodes.getJSONObject(i);
        final JSONObject otherInfoNode = vertexNode.getJSONObject(ATSConstants.OTHER_INFO);
        final String vertexName = otherInfoNode.getString(ATSConstants.VERTEX_NAME);
        final VertexTaskStats vertexTaskStats =
            new VertexTaskStats(otherInfoNode.optInt(ATSConstants.NUM_TASKS),
                otherInfoNode.optInt(ATSConstants.NUM_COMPLETED_TASKS),
                otherInfoNode.optInt(ATSConstants.NUM_SUCCEEDED_TASKS),
                otherInfoNode.optInt(ATSConstants.NUM_KILLED_TASKS),
                otherInfoNode.optInt(ATSConstants.NUM_FAILED_TASKS));
        vertexTaskStatsMap.put(vertexName, vertexTaskStats);
      }
      vertexTaskStatsCache = vertexTaskStatsMap;
    }
  }
  return vertexTaskStatsCache;
}
 
Example 9
Source Project: tez   File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Parse tez counters from json
 *
 * @param jsonObject
 * @return TezCounters
 * @throws JSONException
 */
public static TezCounters parseTezCountersFromJSON(JSONObject jsonObject)
    throws JSONException {
  TezCounters counters = new TezCounters();

  if (jsonObject == null) {
    return counters; //empty counters.
  }

  final JSONArray counterGroupNodes = jsonObject.optJSONArray(Constants.COUNTER_GROUPS);
  if (counterGroupNodes != null) {
    for (int i = 0; i < counterGroupNodes.length(); i++) {
      JSONObject counterGroupNode = counterGroupNodes.optJSONObject(i);
      final String groupName = counterGroupNode.optString(Constants.COUNTER_GROUP_NAME);
      final String groupDisplayName = counterGroupNode.optString(
          Constants.COUNTER_GROUP_DISPLAY_NAME, groupName);

      CounterGroup group = counters.addGroup(groupName, groupDisplayName);

      final JSONArray counterNodes = counterGroupNode.optJSONArray(Constants.COUNTERS);

      //Parse counter nodes
      for (int j = 0; j < counterNodes.length(); j++) {
        JSONObject counterNode = counterNodes.optJSONObject(j);
        final String counterName = counterNode.getString(Constants.COUNTER_NAME);
        final String counterDisplayName =
            counterNode.optString(Constants.COUNTER_DISPLAY_NAME, counterName);
        final long counterValue = counterNode.getLong(Constants.COUNTER_VALUE);
        addCounter(group, counterName, counterDisplayName, counterValue);
      }
    }
  }
  return counters;
}
 
Example 10
Source Project: tez   File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
public static List<DataDependencyEvent> parseDataEventDependencyFromJSON(JSONObject jsonObject) 
    throws JSONException {
  List<DataDependencyEvent> events = Lists.newArrayList();
  JSONArray fields = jsonObject.optJSONArray(Constants.LAST_DATA_EVENTS);
  for (int i=0; i<fields.length(); i++) {
    JSONObject eventMap = fields.getJSONObject(i);
    events.add(new DataDependencyEvent(
        StringInterner.weakIntern(eventMap.optString(EntityTypes.TEZ_TASK_ATTEMPT_ID.name())),
        eventMap.optLong(Constants.TIMESTAMP)));
  }
  return events;
}
 
Example 11
Source Project: tez   File: DagInfo.java    License: Apache License 2.0 5 votes vote down vote up
private void parseDAGPlan(JSONObject dagPlan) throws JSONException {
  int version = dagPlan.optInt(Constants.VERSION, 1);
  parseEdges(dagPlan.optJSONArray(Constants.EDGES));

  JSONArray verticesInfo = dagPlan.optJSONArray(Constants.VERTICES);
  parseBasicVertexInfo(verticesInfo);

  if (version > 1) {
    parseDAGContext(dagPlan.optJSONObject(Constants.DAG_CONTEXT));
  }
}
 
Example 12
Source Project: Bats   File: LogicalPlanSerializer.java    License: Apache License 2.0 4 votes vote down vote up
public static PropertiesConfiguration convertToProperties(JSONObject json) throws JSONException
{
  PropertiesConfiguration props = new PropertiesConfiguration();
  JSONArray allOperators = json.getJSONArray("operators");
  JSONArray allStreams = json.getJSONArray("streams");

  for (int j = 0; j < allOperators.length(); j++) {
    JSONObject operatorDetail = allOperators.getJSONObject(j);
    String operatorName = operatorDetail.getString("name");
    String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorName;
    props.setProperty(operatorKey + ".classname", operatorDetail.getString("class"));
    JSONObject properties = operatorDetail.optJSONObject("properties");
    if (properties != null) {
      Iterator<String> iter2 = properties.keys();
      while (iter2.hasNext()) {
        String propertyName = iter2.next();
        if (!propertyName.equals("name") && !propertyName.equals("class") && properties.opt(propertyName) != null) {
          JSONArray list = properties.optJSONArray(propertyName);
          String value = "";
          if (list != null) {
            for (int i = 0; i < list.length(); i++) {
              if (i != 0) {
                value += ",";
              }
              value += list.get(i).toString();
            }
            props.setProperty(operatorKey + "." + propertyName, value);
          } else {
            props.setProperty(operatorKey + "." + propertyName, properties.get(propertyName));
          }
        }
      }
    }
  }

  for (int j = 0; j < allStreams.length(); j++) {
    JSONObject streamDetail = allStreams.getJSONObject(j);
    String streamName = streamDetail.getString("name");
    String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamName;
    JSONObject sourceDetail = streamDetail.getJSONObject("source");
    JSONArray sinksList = streamDetail.getJSONArray("sinks");

    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, sourceDetail.getString("operatorName") + "." + sourceDetail.getString("portName"));
    String sinksValue = "";
    for (int i = 0; i < sinksList.length(); i++) {
      if (!sinksValue.isEmpty()) {
        sinksValue += ",";
      }
      sinksValue += sinksList.getJSONObject(i).getString("operatorName") + "." + sinksList.getJSONObject(i).getString("portName");
    }
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
    String locality = streamDetail.optString("locality", null);
    if (locality != null) {
      props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, Locality.valueOf(locality));
    }
  }

  // TBD: Attributes

  return props;
}
 
Example 13
private String authenticateADFS(HttpClientAdapter httpClientAdapter, String responseBodyAsString, String authorizeUrl) throws IOException {
    URI location;

    if (responseBodyAsString.contains("login.microsoftonline.com")) {
        LOGGER.info("Already authenticated through Basic or NTLM");
    } else {
        // parse form to get target url, authenticate as userid
        PostRequest logonMethod = new PostRequest(extract("method=\"post\" action=\"([^\"]+)\"", responseBodyAsString));
        logonMethod.setRequestHeader("Content-Type", "application/x-www-form-urlencoded");

        logonMethod.setParameter("UserName", userid);
        logonMethod.setParameter("Password", password);
        logonMethod.setParameter("AuthMethod", "FormsAuthentication");

        httpClientAdapter.executePostRequest(logonMethod);
        location = logonMethod.getRedirectLocation();
        if (location == null) {
            throw new DavMailAuthenticationException("EXCEPTION_AUTHENTICATION_FAILED");
        }

        GetRequest redirectMethod = new GetRequest(location);
        responseBodyAsString = httpClientAdapter.executeGetRequest(redirectMethod);
    }

    if (!responseBodyAsString.contains("login.microsoftonline.com")) {
        throw new DavMailAuthenticationException("EXCEPTION_AUTHENTICATION_FAILED");
    }
    String targetUrl = extract("action=\"([^\"]+)\"", responseBodyAsString);
    String wa = extract("name=\"wa\" value=\"([^\"]+)\"", responseBodyAsString);
    String wresult = extract("name=\"wresult\" value=\"([^\"]+)\"", responseBodyAsString);
    // decode wresult
    wresult = wresult.replaceAll("&quot;", "\"");
    wresult = wresult.replaceAll("&lt;", "<");
    wresult = wresult.replaceAll("&gt;", ">");
    String wctx = extract("name=\"wctx\" value=\"([^\"]+)\"", responseBodyAsString);
    wctx = wctx.replaceAll("&amp;", "&");

    PostRequest targetMethod = new PostRequest(targetUrl);
    targetMethod.setRequestHeader("Content-type", "application/x-www-form-urlencoded");
    targetMethod.setParameter("wa", wa);
    targetMethod.setParameter("wresult", wresult);
    targetMethod.setParameter("wctx", wctx);

    responseBodyAsString = httpClientAdapter.executePostRequest(targetMethod);
    location = targetMethod.getRedirectLocation();

    LOGGER.debug(targetMethod.getURI().toString());
    LOGGER.debug(targetMethod.getReasonPhrase());
    LOGGER.debug(responseBodyAsString);

    if (targetMethod.getStatusCode() == HttpStatus.SC_OK) {
        JSONObject config = extractConfig(responseBodyAsString);
        if (config.optJSONArray("arrScopes") != null || config.optJSONArray("urlPostRedirect") != null) {
            LOGGER.debug("Authentication successful but user consent or validation needed, please open the following url in a browser");
            LOGGER.debug(authorizeUrl);
            throw new DavMailAuthenticationException("EXCEPTION_AUTHENTICATION_FAILED");
        }
    } else if (targetMethod.getStatusCode() != HttpStatus.SC_MOVED_TEMPORARILY || location == null) {
        throw new IOException("Unknown ADFS authentication failure");
    }

    if ("device.login.microsoftonline.com".equals(location.getHost())) {
        location = processDeviceLogin(httpClientAdapter, location);
    }
    String query = location.getQuery();
    if (query == null) {
        // failover for null query with non https URI like urn:ietf:wg:oauth:2.0:oob?code=...
        query = location.getSchemeSpecificPart();
    }

    if (query.contains("code=") && query.contains("&session_state=")) {
        String code = query.substring(query.indexOf("code=") + 5, query.indexOf("&session_state="));
        LOGGER.debug("Authentication Code: " + code);
        return code;
    }
    throw new IOException("Unknown ADFS authentication failure");
}
 
Example 14
public static PropertiesConfiguration convertToProperties(JSONObject json) throws JSONException
{
  PropertiesConfiguration props = new PropertiesConfiguration();
  JSONArray allOperators = json.getJSONArray("operators");
  JSONArray allStreams = json.getJSONArray("streams");

  for (int j = 0; j < allOperators.length(); j++) {
    JSONObject operatorDetail = allOperators.getJSONObject(j);
    String operatorName = operatorDetail.getString("name");
    String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorName;
    props.setProperty(operatorKey + ".classname", operatorDetail.getString("class"));
    JSONObject properties = operatorDetail.optJSONObject("properties");
    if (properties != null) {
      Iterator<String> iter2 = properties.keys();
      while (iter2.hasNext()) {
        String propertyName = iter2.next();
        if (!propertyName.equals("name") && !propertyName.equals("class") && properties.opt(propertyName) != null) {
          JSONArray list = properties.optJSONArray(propertyName);
          String value = "";
          if (list != null) {
            for (int i = 0; i < list.length(); i++) {
              if (i != 0) {
                value += ",";
              }
              value += list.get(i).toString();
            }
            props.setProperty(operatorKey + "." + propertyName, value);
          } else {
            props.setProperty(operatorKey + "." + propertyName, properties.get(propertyName));
          }
        }
      }
    }
  }

  for (int j = 0; j < allStreams.length(); j++) {
    JSONObject streamDetail = allStreams.getJSONObject(j);
    String streamName = streamDetail.getString("name");
    String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamName;
    JSONObject sourceDetail = streamDetail.getJSONObject("source");
    JSONArray sinksList = streamDetail.getJSONArray("sinks");

    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, sourceDetail.getString("operatorName") + "." + sourceDetail.getString("portName"));
    String sinksValue = "";
    for (int i = 0; i < sinksList.length(); i++) {
      if (!sinksValue.isEmpty()) {
        sinksValue += ",";
      }
      sinksValue += sinksList.getJSONObject(i).getString("operatorName") + "." + sinksList.getJSONObject(i).getString("portName");
    }
    props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
    String locality = streamDetail.optString("locality", null);
    if (locality != null) {
      props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, Locality.valueOf(locality));
    }
  }

  // TBD: Attributes

  return props;
}
 
Example 15
Source Project: tez   File: SimpleHistoryParser.java    License: Apache License 2.0 4 votes vote down vote up
protected void readEventsFromSource(String dagId, JSONObjectSource source,
    Map<String, JSONObject> vertexJsonMap, Map<String, JSONObject> taskJsonMap,
    Map<String, JSONObject> attemptJsonMap) throws JSONException, TezException, IOException{
  JSONObject dagJson = null;
  TezDAGID tezDAGID = TezDAGID.fromString(dagId);
  String userName = null;

  while (source.hasNext()) {
    JSONObject jsonObject = source.next();

    String entity = jsonObject.getString(Constants.ENTITY);
    String entityType = jsonObject.getString(Constants.ENTITY_TYPE);
    switch (entityType) {
    case Constants.TEZ_DAG_ID:
      if (!dagId.equals(entity)) {
        LOG.warn(dagId + " is not matching with " + entity);
        continue;
      }
      // Club all DAG related information together (DAG_INIT, DAG_FINISH etc). Each of them
      // would have a set of entities in otherinfo (e.g vertex mapping, dagPlan, start/finish
      // time etc).
      if (dagJson == null) {
        dagJson = jsonObject;
      } else if (dagJson.optJSONObject(ATSConstants.OTHER_INFO)
          .optJSONObject(ATSConstants.DAG_PLAN) == null) {
        // if DAG_PLAN is not filled already, let's try to fetch it from other
        dagJson.getJSONObject(ATSConstants.OTHER_INFO).put(ATSConstants.DAG_PLAN, jsonObject
            .getJSONObject(ATSConstants.OTHER_INFO).getJSONObject(ATSConstants.DAG_PLAN));
      }
      JSONArray relatedEntities = dagJson.optJSONArray(Constants
          .RELATED_ENTITIES);
      //UserName is present in related entities
      // {"entity":"userXYZ","entitytype":"user"}
      if (relatedEntities != null) {
        for (int i = 0; i < relatedEntities.length(); i++) {
          JSONObject subEntity = relatedEntities.getJSONObject(i);
          String subEntityType = subEntity.optString(Constants.ENTITY_TYPE);
          if (subEntityType != null && subEntityType.equals(Constants.USER)) {
            userName = subEntity.getString(Constants.ENTITY);
            break;
          }
        }
      }
      populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO),
          dagJson.getJSONObject(Constants.OTHER_INFO));
      break;
    case Constants.TEZ_VERTEX_ID:
      String vertexName = entity;
      TezVertexID tezVertexID = TezVertexID.fromString(vertexName);
      if (!tezDAGID.equals(tezVertexID.getDAGId())) {
        LOG.warn("{} does not belong to {} ('{}' != '{}')}", vertexName, tezDAGID, tezDAGID, tezVertexID.getDAGId());
        continue;
      }
      if (!vertexJsonMap.containsKey(vertexName)) {
        vertexJsonMap.put(vertexName, jsonObject);
      }
      populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO), vertexName, vertexJsonMap);
      break;
    case Constants.TEZ_TASK_ID:
      String taskName = entity;
      TezTaskID tezTaskID = TezTaskID.fromString(taskName);
      if (!tezDAGID.equals(tezTaskID.getVertexID().getDAGId())) {
        LOG.warn("{} does not belong to {} ('{}' != '{}')}", taskName, tezDAGID, tezDAGID,
            tezTaskID.getVertexID().getDAGId());
        continue;
      }
      if (!taskJsonMap.containsKey(taskName)) {
        taskJsonMap.put(taskName, jsonObject);
      }
      populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO), taskName, taskJsonMap);
      break;
    case Constants.TEZ_TASK_ATTEMPT_ID:
      String taskAttemptName = entity;
      TezTaskAttemptID tezAttemptId = TezTaskAttemptID.fromString(taskAttemptName);
      if (!tezDAGID.equals(tezAttemptId.getTaskID().getVertexID().getDAGId())) {
        LOG.warn("{} does not belong to {} ('{}' != '{}')}", taskAttemptName, tezDAGID, tezDAGID,
            tezAttemptId.getTaskID().getVertexID().getDAGId());
        continue;
      }
      if (!attemptJsonMap.containsKey(taskAttemptName)) {
        attemptJsonMap.put(taskAttemptName, jsonObject);
      }
      populateOtherInfo(jsonObject.optJSONObject(Constants.OTHER_INFO), taskAttemptName, attemptJsonMap);
      break;
    default:
      break;
    }
  }
  source.close();
  if (dagJson != null) {
    this.dagInfo = DagInfo.create(dagJson);
    setUserName(userName);
  } else {
    LOG.error("Dag is not yet parsed. Looks like partial file.");
    throw new TezException(
        "Please provide a valid/complete history log file containing " + dagId);
  }
}
 
Example 16
Source Project: tez   File: DagInfo.java    License: Apache License 2.0 4 votes vote down vote up
DagInfo(JSONObject jsonObject) throws JSONException {
  super(jsonObject);

  vertexNameMap = Maps.newHashMap();
  vertexNameIDMapping = new DualHashBidiMap<>();
  edgeInfoMap = Maps.newHashMap();
  basicVertexInfoMap = Maps.newHashMap();
  containerMapping = LinkedHashMultimap.create();

  Preconditions.checkArgument(jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase
      (Constants.TEZ_DAG_ID));

  dagId = StringInterner.weakIntern(jsonObject.getString(Constants.ENTITY));

  //Parse additional Info
  JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO);

  long sTime = otherInfoNode.optLong(Constants.START_TIME);
  long eTime= otherInfoNode.optLong(Constants.FINISH_TIME);
  userName = otherInfoNode.optString(Constants.USER);
  if (eTime < sTime) {
    LOG.warn("DAG has got wrong start/end values. "
        + "startTime=" + sTime + ", endTime=" + eTime + ". Will check "
        + "timestamps in DAG started/finished events");

    // Check if events DAG_STARTED, DAG_FINISHED can be made use of
    for(Event event : eventList) {
      switch (HistoryEventType.valueOf(event.getType())) {
      case DAG_STARTED:
        sTime = event.getAbsoluteTime();
        break;
      case DAG_FINISHED:
        eTime = event.getAbsoluteTime();
        break;
      default:
        break;
      }
    }

    if (eTime < sTime) {
      LOG.warn("DAG has got wrong start/end values in events as well. "
          + "startTime=" + sTime + ", endTime=" + eTime);
    }
  }
  startTime = sTime;
  endTime = eTime;

  //TODO: Not getting populated correctly for lots of jobs.  Verify
  submitTime = otherInfoNode.optLong(Constants.START_REQUESTED_TIME);
  diagnostics = otherInfoNode.optString(Constants.DIAGNOSTICS);
  failedTasks = otherInfoNode.optInt(Constants.NUM_FAILED_TASKS);
  JSONObject dagPlan = otherInfoNode.optJSONObject(Constants.DAG_PLAN);
  name = StringInterner.weakIntern((dagPlan != null) ? (dagPlan.optString(Constants.DAG_NAME)) : null);
  if (dagPlan != null) {
    JSONArray vertices = dagPlan.optJSONArray(Constants.VERTICES);
    if (vertices != null) {
      numVertices = vertices.length();
    } else {
      numVertices = 0;
    }
    parseDAGPlan(dagPlan);
  } else {
    numVertices = 0;
  }
  status = StringInterner.weakIntern(otherInfoNode.optString(Constants.STATUS));

  //parse name id mapping
  JSONObject vertexIDMappingJson = otherInfoNode.optJSONObject(Constants.VERTEX_NAME_ID_MAPPING);
  if (vertexIDMappingJson != null) {
    //get vertex name
    for (Map.Entry<String, BasicVertexInfo> entry : basicVertexInfoMap.entrySet()) {
      String vertexId = vertexIDMappingJson.optString(entry.getKey());
      //vertexName --> vertexId
      vertexNameIDMapping.put(entry.getKey(), vertexId);
    }
  }
}
 
Example 17
Source Project: tez   File: DagInfo.java    License: Apache License 2.0 4 votes vote down vote up
private void parseBasicVertexInfo(JSONArray verticesInfo) throws JSONException {
  if (verticesInfo == null) {
    LOG.info("No vertices available.");
    return;
  }

  //Parse basic information available in DAG for vertex and edges
  for (int i = 0; i < verticesInfo.length(); i++) {
    BasicVertexInfo basicVertexInfo = new BasicVertexInfo();

    JSONObject vJson = verticesInfo.getJSONObject(i);
    basicVertexInfo.vertexName =
        vJson.optString(Constants.VERTEX_NAME);
    JSONArray inEdges = vJson.optJSONArray(Constants.IN_EDGE_IDS);
    if (inEdges != null) {
      String[] inEdgeIds = new String[inEdges.length()];
      for (int j = 0; j < inEdges.length(); j++) {
        inEdgeIds[j] = inEdges.get(j).toString();
      }
      basicVertexInfo.inEdgeIds = inEdgeIds;
    }

    JSONArray outEdges = vJson.optJSONArray(Constants.OUT_EDGE_IDS);
    if (outEdges != null) {
      String[] outEdgeIds = new String[outEdges.length()];
      for (int j = 0; j < outEdges.length(); j++) {
        outEdgeIds[j] = outEdges.get(j).toString();
      }
      basicVertexInfo.outEdgeIds = outEdgeIds;
    }

    JSONArray addInputsJson =
        vJson.optJSONArray(Constants.ADDITIONAL_INPUTS);
    basicVertexInfo.additionalInputs = parseAdditionalDetailsForVertex(addInputsJson);

    JSONArray addOutputsJson =
        vJson.optJSONArray(Constants.ADDITIONAL_OUTPUTS);
    basicVertexInfo.additionalOutputs = parseAdditionalDetailsForVertex(addOutputsJson);

    basicVertexInfoMap.put(basicVertexInfo.vertexName, basicVertexInfo);
  }
}
 
Example 18
Source Project: tez   File: ATSFileParser.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Read zip file contents. Every file can contain "dag", "vertices", "tasks", "task_attempts"
 *
 * @param atsFile
 * @throws IOException
 * @throws JSONException
 */
private void parseATSZipFile(File atsFile)
    throws IOException, JSONException, TezException, InterruptedException {
  final ZipFile atsZipFile = new ZipFile(atsFile);
  try {
    Enumeration<? extends ZipEntry> zipEntries = atsZipFile.entries();
    while (zipEntries.hasMoreElements()) {
      ZipEntry zipEntry = zipEntries.nextElement();
      LOG.debug("Processing " + zipEntry.getName());
      InputStream inputStream = atsZipFile.getInputStream(zipEntry);
      JSONObject jsonObject = readJson(inputStream);

      //This json can contain dag, vertices, tasks, task_attempts
      JSONObject dagJson = jsonObject.optJSONObject(Constants.DAG);
      if (dagJson != null) {
        //TODO: support for multiple dags per ATS file later.
        dagInfo = DagInfo.create(dagJson);
      }

      //Process vertex
      JSONArray vertexJson = jsonObject.optJSONArray(Constants.VERTICES);
      if (vertexJson != null) {
        processVertices(vertexJson);
      }

      //Process task
      JSONArray taskJson = jsonObject.optJSONArray(Constants.TASKS);
      if (taskJson != null) {
        processTasks(taskJson);
      }

      //Process task attempts
      JSONArray attemptsJson = jsonObject.optJSONArray(Constants.TASK_ATTEMPTS);
      if (attemptsJson != null) {
        processAttempts(attemptsJson);
      }

      //Process application (mainly versionInfo)
      JSONObject tezAppJson = jsonObject.optJSONObject(Constants.APPLICATION);
      if (tezAppJson != null) {
        processApplication(tezAppJson);
      }
    }
  } finally {
    atsZipFile.close();
  }
}