org.apache.tez.common.counters.CounterGroup Java Examples

@Override
public void initialize(ProcessorContext processorContext)
        throws ExecException {
    if (isMultiStore()) {
        CounterGroup multiStoreGroup = processorContext.getCounters()
                .getGroup(MRPigStatsUtil.MULTI_STORE_COUNTER_GROUP);
        if (multiStoreGroup == null) {
            processorContext.getCounters().addGroup(
                    MRPigStatsUtil.MULTI_STORE_COUNTER_GROUP,
                    MRPigStatsUtil.MULTI_STORE_COUNTER_GROUP);
        }
        String name = MRPigStatsUtil.getMultiStoreCounterName(this);
        if (name != null) {
            outputRecordCounter = multiStoreGroup.addCounter(name, name, 0);
        }
    }
}
 

public static Counters fromTez(TezCounters tezCounters) {
  if (tezCounters == null) {
    return null;
  }
  Counters counters = new Counters();
  for (CounterGroup xGrp : tezCounters) {
    counters.addGroup(xGrp.getName(), xGrp.getDisplayName());
    for (TezCounter xCounter : xGrp) {
      Counter counter =
          counters.findCounter(xGrp.getName(), xCounter.getName());
      counter.setValue(xCounter.getValue());

    }
  }
  return counters;
}
 

public static Map<String,Object> convertCountersToATSMap(TezCounters counters) {
  Map<String,Object> object = new LinkedHashMap<String, Object>();
  if (counters == null) {
      return object;
    }
  ArrayList<Object> counterGroupsList = new ArrayList<Object>();
  for (CounterGroup group : counters) {
      Map<String,Object> counterGroupMap = new LinkedHashMap<String, Object>();
      counterGroupMap.put(ATSConstants.COUNTER_GROUP_NAME, group.getName());
      counterGroupMap.put(ATSConstants.COUNTER_GROUP_DISPLAY_NAME,
              group.getDisplayName());
      ArrayList<Object> counterList = new ArrayList<Object>();
      for (TezCounter counter : group) {
          Map<String,Object> counterMap = new LinkedHashMap<String, Object>();
          counterMap.put(ATSConstants.COUNTER_NAME, counter.getName());
          counterMap.put(ATSConstants.COUNTER_DISPLAY_NAME,
                  counter.getDisplayName());
          counterMap.put(ATSConstants.COUNTER_VALUE, counter.getValue());
          counterList.add(counterMap);
        }
      putInto(counterGroupMap, ATSConstants.COUNTERS, counterList);
      counterGroupsList.add(counterGroupMap);
    }
  putInto(object, ATSConstants.COUNTER_GROUPS, counterGroupsList);
  return object;
}
 

public static Counters fromTez(TezCounters tezCounters) {
  if (tezCounters == null) {
    return null;
  }
  Counters counters = new Counters();
  for (CounterGroup xGrp : tezCounters) {
    counters.addGroup(xGrp.getName(), xGrp.getDisplayName());
    for (TezCounter xCounter : xGrp) {
      Counter counter =
          counters.findCounter(xGrp.getName(), xCounter.getName());
      counter.setValue(xCounter.getValue());

    }
  }
  return counters;
}
 

/**
 * Get counter for a specific counter group name.
 * If counterGroupName is not mentioned, it would end up returning counter found in all
 * groups
 *
 * @param counterGroupName
 * @param counter
 * @return Map<String, TezCounter> tez counter at every counter group level
 */
public Map<String, TezCounter> getCounter(String counterGroupName, String counter) {
  //TODO: FS, TaskCounters are directly getting added as TezCounters always pass those.  Need a
  // way to get rid of these.
  Map<String, TezCounter> result = Maps.newHashMap();
  Iterator<String> iterator = tezCounters.getGroupNames().iterator();
  boolean found = false;
  while (iterator.hasNext()) {
    CounterGroup counterGroup = tezCounters.getGroup(iterator.next());
    if (counterGroupName != null) {
      String groupName = counterGroup.getName();
      if (groupName.equals(counterGroupName)) {
        found = true;
      }
    }

    //Explicitly mention that no need to create the counter if not present
    TezCounter tezCounter = counterGroup.getUnderlyingGroup().findCounter(counter, false);
    if (tezCounter != null) {
      result.put(counterGroup.getName(), tezCounter);
    }

    if (found) {
      //Retrieved counter specific to a counter group. Safe to exit.
      break;
    }

  }
  return result;
}
 

private static void addCounter(CounterGroup group, String counterName, String displayName,
    long counterValue) {
  try {
    TezCounter counter = group.findCounter(counterName, displayName);
    counter.setValue(counterValue);
  } catch(IllegalArgumentException e) {
    LOG.debug("Error finding {} in {} with displayName {}", counterName, group, displayName);
  }
}
 

/**
 * Parse tez counters from json
 *
 * @param jsonObject
 * @return TezCounters
 * @throws JSONException
 */
public static TezCounters parseTezCountersFromJSON(JSONObject jsonObject)
    throws JSONException {
  TezCounters counters = new TezCounters();

  if (jsonObject == null) {
    return counters; //empty counters.
  }

  final JSONArray counterGroupNodes = jsonObject.optJSONArray(Constants.COUNTER_GROUPS);
  if (counterGroupNodes != null) {
    for (int i = 0; i < counterGroupNodes.length(); i++) {
      JSONObject counterGroupNode = counterGroupNodes.optJSONObject(i);
      final String groupName = counterGroupNode.optString(Constants.COUNTER_GROUP_NAME);
      final String groupDisplayName = counterGroupNode.optString(
          Constants.COUNTER_GROUP_DISPLAY_NAME, groupName);

      CounterGroup group = counters.addGroup(groupName, groupDisplayName);

      final JSONArray counterNodes = counterGroupNode.optJSONArray(Constants.COUNTERS);

      //Parse counter nodes
      for (int j = 0; j < counterNodes.length(); j++) {
        JSONObject counterNode = counterNodes.optJSONObject(j);
        final String counterName = counterNode.getString(Constants.COUNTER_NAME);
        final String counterDisplayName =
            counterNode.optString(Constants.COUNTER_DISPLAY_NAME, counterName);
        final long counterValue = counterNode.getLong(Constants.COUNTER_VALUE);
        addCounter(group, counterName, counterDisplayName, counterValue);
      }
    }
  }
  return counters;
}
 

public static TezCountersProto convertTezCountersToProto(
    TezCounters counters) {
  TezCountersProto.Builder builder = TezCountersProto.newBuilder();
  Iterator<CounterGroup> groupIterator = counters.iterator();
  int groupIndex = 0;
  while (groupIterator.hasNext()) {
    CounterGroup counterGroup = groupIterator.next();
    TezCounterGroupProto.Builder groupBuilder =
      TezCounterGroupProto.newBuilder();
    groupBuilder.setName(counterGroup.getName());
    groupBuilder.setDisplayName(counterGroup.getDisplayName());
    Iterator<TezCounter> counterIterator = counterGroup.iterator();
    int counterIndex = 0;
    while (counterIterator.hasNext()) {
      TezCounter counter = counterIterator.next();
      TezCounterProto tezCounterProto = TezCounterProto.newBuilder()
        .setName(counter.getName())
        .setDisplayName(counter.getDisplayName())
        .setValue(counter.getValue())
        .build();
      groupBuilder.addCounters(counterIndex, tezCounterProto);
      ++counterIndex;
    }
    builder.addCounterGroups(groupIndex, groupBuilder.build());
    ++groupIndex;
  }
  return builder.build();
}
 

public static TezCounters convertTezCountersFromProto(TezCountersProto proto) {
  TezCounters counters = new TezCounters();
  for (TezCounterGroupProto counterGroupProto : proto.getCounterGroupsList()) {
    CounterGroup group = counters.addGroup(counterGroupProto.getName(),
      counterGroupProto.getDisplayName());
    for (TezCounterProto counterProto :
      counterGroupProto.getCountersList()) {
      TezCounter counter = group.findCounter(
        counterProto.getName(),
        counterProto.getDisplayName());
      counter.setValue(counterProto.getValue());
    }
  }
  return counters;
}
 

@Override
public String toString() {
  StringBuilder sb = new StringBuilder();
  sb.append("vertexName=");
  sb.append(vertexName);
  sb.append(", taskId=");
  sb.append(taskID);
  sb.append(", startTime=");
  sb.append(startTime);
  sb.append(", finishTime=");
  sb.append(finishTime);
  sb.append(", timeTaken=");
  sb.append(finishTime - startTime);
  sb.append(", status=");
  sb.append(state.name());
  sb.append(", successfulAttemptID=");
  sb.append(successfulAttemptID);
  sb.append(", diagnostics=");
  sb.append(diagnostics);
  sb.append(", counters=");
  if (tezCounters == null) {
    sb.append("null");
  } else {
    sb.append("Counters: ");
    sb.append(tezCounters.countCounters());
    for (CounterGroup group : tezCounters) {
      sb.append(", ");
      sb.append(group.getDisplayName());
      for (TezCounter counter : group) {
        sb.append(", ");
        sb.append(counter.getDisplayName()).append("=")
            .append(counter.getValue());
      }
    }
  }
  return sb.toString();
}
 

public static TezCountersProto convertTezCountersToProto(
    TezCounters counters) {
  TezCountersProto.Builder builder = TezCountersProto.newBuilder();
  Iterator<CounterGroup> groupIterator = counters.iterator();
  int groupIndex = 0;
  while (groupIterator.hasNext()) {
    CounterGroup counterGroup = groupIterator.next();
    TezCounterGroupProto.Builder groupBuilder =
      TezCounterGroupProto.newBuilder();
    groupBuilder.setName(counterGroup.getName());
    groupBuilder.setDisplayName(counterGroup.getDisplayName());
    Iterator<TezCounter> counterIterator = counterGroup.iterator();
    int counterIndex = 0;
    while (counterIterator.hasNext()) {
      TezCounter counter = counterIterator.next();
      TezCounterProto tezCounterProto = TezCounterProto.newBuilder()
        .setName(counter.getName())
        .setDisplayName(counter.getDisplayName())
        .setValue(counter.getValue())
        .build();
      groupBuilder.addCounters(counterIndex, tezCounterProto);
      ++counterIndex;
    }
    builder.addCounterGroups(groupIndex, groupBuilder.build());
    ++groupIndex;
  }
  return builder.build();
}
 

public static TezCounters convertTezCountersFromProto(TezCountersProto proto) {
  TezCounters counters = new TezCounters();
  for (TezCounterGroupProto counterGroupProto : proto.getCounterGroupsList()) {
    CounterGroup group = counters.addGroup(counterGroupProto.getName(),
      counterGroupProto.getDisplayName());
    for (TezCounterProto counterProto :
      counterGroupProto.getCountersList()) {
      TezCounter counter = group.findCounter(
        counterProto.getName(),
        counterProto.getDisplayName());
      counter.setValue(counterProto.getValue());
    }
  }
  return counters;
}
 

private Counters covertToHadoopCounters(TezCounters tezCounters) {
    Counters counters = new Counters();
    for (CounterGroup tezGrp : tezCounters) {
        Group grp = counters.addGroup(tezGrp.getName(), tezGrp.getDisplayName());
        for (TezCounter counter : tezGrp) {
            grp.addCounter(counter.getName(), counter.getDisplayName(), counter.getValue());
        }
    }
    return counters;
}
 

public static Map<String,Object> convertCountersToATSMap(TezCounters counters) {
  Map<String, Object> object = new LinkedHashMap<String, Object>();
  if (counters == null) {
      return object;
    }
  ArrayList<Object> counterGroupsList = new ArrayList<Object>();
  for (CounterGroup group : counters) {
      ArrayList<Object> counterList = new ArrayList<Object>();
      for (TezCounter counter : group) {
        if (counter.getValue() != 0) {
          Map<String,Object> counterMap = new LinkedHashMap<String, Object>();
          counterMap.put(ATSConstants.COUNTER_NAME, counter.getName());
          if (!counter.getDisplayName().equals(counter.getName())) {
            counterMap.put(ATSConstants.COUNTER_DISPLAY_NAME,
                  counter.getDisplayName());
          }
          counterMap.put(ATSConstants.COUNTER_VALUE, counter.getValue());
          if (counter instanceof AggregateTezCounter) {
            counterMap.put(ATSConstants.COUNTER_INSTANCE_COUNT,
                ((AggregateTezCounter)counter).getCount());
            counterMap.put(ATSConstants.COUNTER_MAX_VALUE,
                  ((AggregateTezCounter)counter).getMax());
            counterMap.put(ATSConstants.COUNTER_MIN_VALUE,
                ((AggregateTezCounter)counter).getMin());

          }
          counterList.add(counterMap);
        }
      }
      if (!counterList.isEmpty()) {
        Map<String,Object> counterGroupMap = new LinkedHashMap<String, Object>();
        counterGroupMap.put(ATSConstants.COUNTER_GROUP_NAME, group.getName());
        if (!group.getDisplayName().equals(group.getName())) {
          counterGroupMap.put(ATSConstants.COUNTER_GROUP_DISPLAY_NAME,
              group.getDisplayName());
        }
        counterGroupMap.put(ATSConstants.COUNTERS, counterList);
        counterGroupsList.add(counterGroupMap);
      }
    }
  putInto(object, ATSConstants.COUNTER_GROUPS, counterGroupsList);
  return object;
}
 

@Override
public String toString() {
  StringBuilder sb = new StringBuilder();
  sb.append("vertexName=");
  sb.append(vertexName);
  sb.append(", taskAttemptId=");
  sb.append(taskAttemptId);
  sb.append(", creationTime=");
  sb.append(creationTime);
  sb.append(", allocationTime=");
  sb.append(allocationTime);
  sb.append(", startTime=");
  sb.append(startTime);
  sb.append(", finishTime=");
  sb.append(finishTime);
  sb.append(", timeTaken=");
  sb.append(finishTime - startTime);
  sb.append(", status=");
  sb.append(state.name());

  if (taskFailureType != null) {
    sb.append(", taskFailureType=");
    sb.append(taskFailureType);
  }
  if (error != null) {
    sb.append(", errorEnum=");
    sb.append(error);
  }
  if (diagnostics != null) {
    sb.append(", diagnostics=");
    sb.append(diagnostics);
  }
  if (containerId != null) {
    sb.append(", containerId=");
    sb.append(containerId);
  }
  if (nodeId != null) {
    sb.append(", nodeId=");
    sb.append(nodeId);
  }
  if (nodeHttpAddress != null) {
    sb.append(", nodeHttpAddress=");
    sb.append(nodeHttpAddress);
  }

  if (state != TaskAttemptState.SUCCEEDED) {
    sb.append(", counters=");
    if (tezCounters == null) {
      sb.append("null");
    } else {
      sb.append("Counters: ");
      sb.append(tezCounters.countCounters());
      for (CounterGroup group : tezCounters) {
        sb.append(", ");
        sb.append(group.getDisplayName());
        for (TezCounter counter : group) {
          sb.append(", ");
          sb.append(counter.getDisplayName()).append("=")
              .append(counter.getValue());
        }
      }
    }
  }
  return sb.toString();
}
 

@Test(timeout = 60000)
public void testPerIOCounterAggregation() throws Exception {
  String baseDir = "/tmp/perIOCounterAgg/";
  Path inPath1 = new Path(baseDir + "inPath1");
  Path inPath2 = new Path(baseDir + "inPath2");
  Path outPath = new Path(baseDir + "outPath");
  final Set<String> expectedResults = generateSortMergeJoinInput(inPath1, inPath2);
  Path stagingDirPath = new Path("/tmp/tez-staging-dir");
  remoteFs.mkdirs(stagingDirPath);

  TezConfiguration conf = new TezConfiguration(mrrTezCluster.getConfig());
  conf.setBoolean(TezConfiguration.TEZ_TASK_GENERATE_COUNTERS_PER_IO, true);
  TezClient tezClient = TezClient.create(SortMergeJoinHelper.class.getSimpleName(), conf);
  tezClient.start();

  SortMergeJoinHelper sortMergeJoinHelper = new SortMergeJoinHelper(tezClient);
  sortMergeJoinHelper.setConf(conf);

  String[] args = new String[] {
      "-D" + TezConfiguration.TEZ_AM_STAGING_DIR + "=" + stagingDirPath.toString(),
      "-counter", inPath1.toString(), inPath2.toString(), "1", outPath.toString() };
  assertEquals(0, sortMergeJoinHelper.run(conf, args, tezClient));

  verifySortMergeJoinInput(outPath, expectedResults);

  String joinerVertexName = "joiner";
  String input1Name = "input1";
  String input2Name = "input2";
  String joinOutputName = "joinOutput";
  Set<StatusGetOpts> statusOpts = new HashSet<StatusGetOpts>();
  statusOpts.add(StatusGetOpts.GET_COUNTERS);
  VertexStatus joinerVertexStatus =
      sortMergeJoinHelper.dagClient.getVertexStatus(joinerVertexName, statusOpts);
  final TezCounters joinerCounters = joinerVertexStatus.getVertexCounters();
  final CounterGroup aggregatedGroup = joinerCounters.getGroup(TaskCounter.class.getCanonicalName());
  final CounterGroup input1Group = joinerCounters.getGroup(
      TaskCounter.class.getSimpleName() + "_" + joinerVertexName + "_INPUT_" + input1Name);
  final CounterGroup input2Group = joinerCounters.getGroup(
      TaskCounter.class.getSimpleName() + "_" + joinerVertexName + "_INPUT_" + input2Name);
  assertTrue("aggregated counter group cannot be empty", aggregatedGroup.size() > 0);
  assertTrue("per io group for input1 cannot be empty", input1Group.size() > 0);
  assertTrue("per io group for input1 cannot be empty", input2Group.size() > 0);

  List<TaskCounter> countersToVerifyAgg = Arrays.asList(
      TaskCounter.ADDITIONAL_SPILLS_BYTES_READ,
      TaskCounter.ADDITIONAL_SPILLS_BYTES_WRITTEN,
      TaskCounter.COMBINE_INPUT_RECORDS,
      TaskCounter.MERGED_MAP_OUTPUTS,
      TaskCounter.NUM_DISK_TO_DISK_MERGES,
      TaskCounter.NUM_FAILED_SHUFFLE_INPUTS,
      TaskCounter.NUM_MEM_TO_DISK_MERGES,
      TaskCounter.NUM_SHUFFLED_INPUTS,
      TaskCounter.NUM_SKIPPED_INPUTS,
      TaskCounter.REDUCE_INPUT_GROUPS,
      TaskCounter.REDUCE_INPUT_RECORDS,
      TaskCounter.SHUFFLE_BYTES,
      TaskCounter.SHUFFLE_BYTES_DECOMPRESSED,
      TaskCounter.SHUFFLE_BYTES_DISK_DIRECT,
      TaskCounter.SHUFFLE_BYTES_TO_DISK,
      TaskCounter.SHUFFLE_BYTES_TO_MEM,
      TaskCounter.SPILLED_RECORDS
  );

  int nonZeroCounters = 0;
  // verify that the sum of the counter values for edges add up to the aggregated counter value.
  for(TaskCounter c : countersToVerifyAgg) {
    TezCounter aggregatedCounter = aggregatedGroup.findCounter(c.name(), false);
    TezCounter input1Counter = input1Group.findCounter(c.name(), false);
    TezCounter input2Counter = input2Group.findCounter(c.name(), false);
    assertNotNull("aggregated counter cannot be null " + c.name(), aggregatedCounter);
    assertNotNull("input1 counter cannot be null " + c.name(), input1Counter);
    assertNotNull("input2 counter cannot be null " + c.name(), input2Counter);

    assertEquals("aggregated counter does not match sum of input counters " + c.name(),
        aggregatedCounter.getValue(), input1Counter.getValue() + input2Counter.getValue());

    if (aggregatedCounter.getValue() > 0) {
      nonZeroCounters++;
    }
  }

  // ensure that at least one of the counters tested above were non-zero.
  assertTrue("At least one of the counter should be non-zero. invalid test ", nonZeroCounters > 0);

  CounterGroup joinerOutputGroup = joinerCounters.getGroup(
      TaskCounter.class.getSimpleName() + "_" + joinerVertexName + "_OUTPUT_" + joinOutputName);
  String outputCounterName = TaskCounter.OUTPUT_RECORDS.name();
  TezCounter aggregateCounter = aggregatedGroup.findCounter(outputCounterName, false);
  TezCounter joinerOutputCounter = joinerOutputGroup.findCounter(outputCounterName, false);
  assertNotNull("aggregated counter cannot be null " + outputCounterName, aggregateCounter);
  assertNotNull("output counter cannot be null " + outputCounterName, joinerOutputCounter);
  assertTrue("counter value is zero. test is invalid", aggregateCounter.getValue() > 0);
  assertEquals("aggregated counter does not match sum of output counters " + outputCounterName,
      aggregateCounter.getValue(), joinerOutputCounter.getValue());
}
 

@Test (timeout = 10000)
public void testBasicCounters() throws Exception {
  TezConfiguration tezconf = new TezConfiguration(defaultConf);
  MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null,
      null, false, false);
  tezClient.start();

  final String vAName = "A";
  final String vBName = "B";
  final String procCounterName = "Proc";
  final String globalCounterName = "Global";
  DAG dag = DAG.create("testBasicCounters");
  Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 10);
  Vertex vB = Vertex.create(vBName, ProcessorDescriptor.create("Proc.class"), 1);
  dag.addVertex(vA)
      .addVertex(vB)
      .addEdge(
          Edge.create(vA, vB, EdgeProperty.create(DataMovementType.SCATTER_GATHER,
              DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
              OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
  TezCounters temp = new TezCounters();
  temp.findCounter(new String(globalCounterName), new String(globalCounterName)).increment(1);
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  DataOutput out = new DataOutputStream(bos);
  temp.write(out);
  final byte[] payload = bos.toByteArray();

  MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
  MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
  mockLauncher.startScheduling(false);
  mockApp.countersDelegate = new CountersDelegate() {
    @Override
    public TezCounters getCounters(TaskSpec taskSpec) {
      String vName = taskSpec.getVertexName();
      TezCounters counters = new TezCounters();
      final DataInputByteBuffer in  = new DataInputByteBuffer();
      in.reset(ByteBuffer.wrap(payload));
      try {
        // this ensures that the serde code path is covered.
        // the internal merges of counters covers the constructor code path.
        counters.readFields(in);
      } catch (IOException e) {
        Assert.fail(e.getMessage());
      }
      counters.findCounter(vName, procCounterName).increment(1);
      for (OutputSpec output : taskSpec.getOutputs()) {
        counters.findCounter(vName, output.getDestinationVertexName()).increment(1);
      }
      for (InputSpec input : taskSpec.getInputs()) {
        counters.findCounter(vName, input.getSourceVertexName()).increment(1);
      }
      return counters;
    }
  };
  mockApp.doSleep = false;
  DAGClient dagClient = tezClient.submitDAG(dag);
  mockLauncher.waitTillContainersLaunched();
  DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
  mockLauncher.startScheduling(true);
  DAGStatus status = dagClient.waitForCompletion();
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
  TezCounters counters = dagImpl.getAllCounters();

  String osName = System.getProperty("os.name").toLowerCase(Locale.ENGLISH);
  if (SystemUtils.IS_OS_LINUX) {
    Assert.assertTrue(counters.findCounter(DAGCounter.AM_CPU_MILLISECONDS).getValue() > 0);
  }

  // verify processor counters
  Assert.assertEquals(10, counters.findCounter(vAName, procCounterName).getValue());
  Assert.assertEquals(1, counters.findCounter(vBName, procCounterName).getValue());
  // verify edge counters
  Assert.assertEquals(10, counters.findCounter(vAName, vBName).getValue());
  Assert.assertEquals(1, counters.findCounter(vBName, vAName).getValue());
  // verify global counters
  Assert.assertEquals(11, counters.findCounter(globalCounterName, globalCounterName).getValue());
  VertexImpl vAImpl = (VertexImpl) dagImpl.getVertex(vAName);
  VertexImpl vBImpl = (VertexImpl) dagImpl.getVertex(vBName);
  TezCounters vACounters = vAImpl.getAllCounters();
  TezCounters vBCounters = vBImpl.getAllCounters();
  String vACounterName = vACounters.findCounter(globalCounterName, globalCounterName).getName();
  String vBCounterName = vBCounters.findCounter(globalCounterName, globalCounterName).getName();
  if (vACounterName != vBCounterName) {
    Assert.fail("String counter name objects dont match despite interning.");
  }
  CounterGroup vaGroup = vACounters.getGroup(globalCounterName);
  String vaGrouName = vaGroup.getName();
  CounterGroup vBGroup = vBCounters.getGroup(globalCounterName);
  String vBGrouName = vBGroup.getName();
  if (vaGrouName != vBGrouName) {
    Assert.fail("String group name objects dont match despite interning.");
  }
  
  tezClient.stop();
}
 

public void accumulateStats(VertexStatus status, int parallelism) {
    hdfsBytesRead = -1;
    hdfsBytesWritten = -1;

    if (status != null) {
        setSuccessful(status.getState().equals(VertexStatus.State.SUCCEEDED));
        this.parallelism = parallelism;
        if (this.isMapOpts) {
            numberMaps += parallelism;
        } else {
            numberReduces += parallelism;
        }
        TezCounters tezCounters = status.getVertexCounters();
        counters = Maps.newHashMap();
        Iterator<CounterGroup> grpIt = tezCounters.iterator();
        while (grpIt.hasNext()) {
            CounterGroup grp = grpIt.next();
            Iterator<TezCounter> cntIt = grp.iterator();
            Map<String, Long> cntMap = Maps.newHashMap();
            while (cntIt.hasNext()) {
                TezCounter cnt = cntIt.next();
                cntMap.put(cnt.getName(), cnt.getValue());
            }
            counters.put(grp.getName(), cntMap);
        }

        if (counters.get(FS_COUNTER_GROUP) != null &&
                counters.get(FS_COUNTER_GROUP).get(PigStatsUtil.HDFS_BYTES_READ) != null) {
            hdfsBytesRead = counters.get(FS_COUNTER_GROUP).get(PigStatsUtil.HDFS_BYTES_READ);
        }
        if (counters.get(FS_COUNTER_GROUP) != null &&
                counters.get(FS_COUNTER_GROUP).get(PigStatsUtil.HDFS_BYTES_WRITTEN) != null) {
            hdfsBytesWritten = counters.get(FS_COUNTER_GROUP).get(PigStatsUtil.HDFS_BYTES_WRITTEN);
        }
        Map<String, Long> pigCounters = counters.get(PIG_COUNTER_GROUP);
        if (pigCounters != null) {
            if (pigCounters.containsKey(PigCounters.SPILLABLE_MEMORY_MANAGER_SPILL_COUNT)) {
                spillCount = pigCounters.get(PigCounters.SPILLABLE_MEMORY_MANAGER_SPILL_COUNT);
            }
            if (pigCounters.containsKey(PigCounters.PROACTIVE_SPILL_COUNT_BAGS)) {
                activeSpillCountObj = pigCounters.get(PigCounters.PROACTIVE_SPILL_COUNT_BAGS);
            }
            if (pigCounters.containsKey(PigCounters.PROACTIVE_SPILL_COUNT_RECS)) {
                activeSpillCountRecs = pigCounters.get(PigCounters.PROACTIVE_SPILL_COUNT_RECS);
            }
        }

        addInputStatistics();
        addOutputStatistics();
    }
}
 

/**
 * Updates the statistics after a DAG is finished.
 */
public void accumulateStats(TezJob tezJob) throws IOException {
    //For Oozie Pig action job id matching compatibility with MR mode
    appId = tezJob.getApplicationId().toString().replace("application", "job");
    setConf(tezJob.getConfiguration());
    DAG dag = tezJob.getDAG();
    hdfsBytesRead = -1;
    hdfsBytesWritten = -1;
    TezCounters tezCounters = tezJob.getDAGCounters();
    if (tezCounters != null) {
        counters = covertToHadoopCounters(tezCounters);

        CounterGroup dagGrp = tezCounters.getGroup(DAG_COUNTER_GROUP);
        totalTasks = (int) dagGrp.findCounter("TOTAL_LAUNCHED_TASKS").getValue();

        CounterGroup fsGrp = tezCounters.getGroup(FS_COUNTER_GROUP);
        fileBytesRead = fsGrp.findCounter("FILE_BYTES_READ").getValue();
        fileBytesWritten = fsGrp.findCounter("FILE_BYTES_WRITTEN").getValue();
        hdfsBytesRead = fsGrp.findCounter("HDFS_BYTES_READ").getValue();
        hdfsBytesWritten = fsGrp.findCounter("HDFS_BYTES_WRITTEN").getValue();
    } else {
        LOG.warn("Failed to get counters for DAG: " + dag.getName());
    }

    for (Entry<String, TezVertexStats> entry : tezVertexStatsMap.entrySet()) {
        Vertex v = dag.getVertex(entry.getKey());
        if (v != null && tezVertexStatsMap.containsKey(v.getName())) {
            TezVertexStats vertexStats = entry.getValue();
            UserPayload payload = v.getProcessorDescriptor().getUserPayload();
            Configuration conf = TezUtils.createConfFromUserPayload(payload);
            vertexStats.setConf(conf);

            VertexStatus status = tezJob.getVertexStatus(v.getName());
            if (status == null) {
                LOG.warn("Failed to get status for vertex: " + v.getName());
                continue;
            }
            vertexStats.accumulateStats(status, v.getParallelism());
            if(vertexStats.getInputs() != null && !vertexStats.getInputs().isEmpty()) {
                inputs.addAll(vertexStats.getInputs());
            }
            if(vertexStats.getOutputs() != null  && !vertexStats.getOutputs().isEmpty()) {
                outputs.addAll(vertexStats.getOutputs());
            }
            /*if (vertexStats.getHdfsBytesRead() >= 0) {
                hdfsBytesRead = (hdfsBytesRead == -1) ? 0 : hdfsBytesRead;
                hdfsBytesRead += vertexStats.getHdfsBytesRead();
            }
            if (vertexStats.getHdfsBytesWritten() >= 0) {
                hdfsBytesWritten = (hdfsBytesWritten == -1) ? 0 : hdfsBytesWritten;
                hdfsBytesWritten += vertexStats.getHdfsBytesWritten();
            }*/
            if (!vertexStats.getMultiStoreCounters().isEmpty()) {
                multiStoreCounters.putAll(vertexStats.getMultiStoreCounters());
            }
            numberMaps += vertexStats.getNumberMaps();
            numberReduces += vertexStats.getNumberReduces();
            mapInputRecords += vertexStats.getMapInputRecords();
            mapOutputRecords += vertexStats.getMapOutputRecords();
            reduceInputRecords += vertexStats.getReduceInputRecords();
            reduceOutputRecords += vertexStats.getReduceOutputRecords();
            spillCount += vertexStats.getSMMSpillCount();
            activeSpillCountObj  += vertexStats.getProactiveSpillCountObjects();
            activeSpillCountRecs += vertexStats.getProactiveSpillCountRecs();
        }
    }
}