Java Code Examples for org.apache.hadoop.mapreduce.CounterGroup#findCounter()
The following examples show how to use
org.apache.hadoop.mapreduce.CounterGroup#findCounter() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: CounterGroupInfo.java From hadoop with Apache License 2.0 | 5 votes |
|
public CounterGroupInfo(String name, CounterGroup group, CounterGroup mg,
CounterGroup rg) {
this.counterGroupName = name;
this.counter = new ArrayList<CounterInfo>();
for (Counter c : group) {
Counter mc = mg == null ? null : mg.findCounter(c.getName());
Counter rc = rg == null ? null : rg.findCounter(c.getName());
CounterInfo cinfo = new CounterInfo(c, mc, rc);
this.counter.add(cinfo);
}
}
Example 2
| Source File: CounterGroupInfo.java From big-c with Apache License 2.0 | 5 votes |
|
public CounterGroupInfo(String name, CounterGroup group, CounterGroup mg,
CounterGroup rg) {
this.counterGroupName = name;
this.counter = new ArrayList<CounterInfo>();
for (Counter c : group) {
Counter mc = mg == null ? null : mg.findCounter(c.getName());
Counter rc = rg == null ? null : rg.findCounter(c.getName());
CounterInfo cinfo = new CounterInfo(c, mc, rc);
this.counter.add(cinfo);
}
}
Example 3
| Source File: YarnJobStatsUtility.java From jumbune with GNU Lesser General Public License v3.0 | 5 votes |
|
/**
* Adds detail for a Map phase.
* @param task2
*
* @param task2 the tasks
* @param referencedZeroTime
* @param referencedZeroTime the start time
* @return the phase details
*/
private TaskOutputDetails addMapPhaseDetails(Entry<TaskAttemptID, TaskAttemptInfo> task, long referencedZeroTime) {
TaskAttemptInfo taskAttemptInfo = (TaskAttemptInfo) (task.getValue());
TaskOutputDetails taskOutputDetails = new TaskOutputDetails();
if(taskAttemptInfo.getTaskStatus().equalsIgnoreCase("SUCCEEDED")){
taskOutputDetails.setTaskStatus(taskAttemptInfo.getTaskStatus());
taskOutputDetails.setTaskType(taskAttemptInfo.getTaskType().toString());
taskOutputDetails.setTaskID(taskAttemptInfo.getAttemptId().getTaskID().toString());
long startPoint = (taskAttemptInfo.getStartTime() - referencedZeroTime) / CONVERSION_FACTOR_MILLISECS_TO_SECS;
taskOutputDetails.setStartPoint(startPoint);
long endPoint = (taskAttemptInfo.getFinishTime() - referencedZeroTime) / CONVERSION_FACTOR_MILLISECS_TO_SECS;
taskOutputDetails.setEndPoint(endPoint);
taskOutputDetails.setTimeTaken(endPoint - startPoint);
taskOutputDetails.setLocation(taskAttemptInfo.getHostname());
Counters counters = taskAttemptInfo.getCounters();
CounterGroup fileSystemCounters = counters.getGroup("org.apache.hadoop.mapreduce.FileSystemCounter");
Counter inputBytes = fileSystemCounters.findCounter("HDFS_BYTES_READ");
long dataFlowRate = inputBytes.getValue() / (endPoint - startPoint);
taskOutputDetails.setDataFlowRate(dataFlowRate);
CounterGroup mapReduceTaskCounters = counters.getGroup("org.apache.hadoop.mapreduce.TaskCounter");
Counter mapOutputRecords = mapReduceTaskCounters.findCounter("MAP_OUTPUT_RECORDS");
Counter physicalMemoryBytes = mapReduceTaskCounters.findCounter("PHYSICAL_MEMORY_BYTES");
ResourceUsageMetrics rum = new ResourceUsageMetrics();
rum.setPhysicalMemoryUsage(physicalMemoryBytes.getValue());
taskOutputDetails.setResourceUsageMetrics(rum);
taskOutputDetails.setOutputRecords(mapOutputRecords.getValue());
Counter mapOutputBytes = mapReduceTaskCounters.findCounter("MAP_OUTPUT_BYTES");
taskOutputDetails.setOutputBytes(mapOutputBytes.getValue());}
return taskOutputDetails;
}
Example 4
| Source File: CountersBlock.java From hadoop with Apache License 2.0 | 4 votes |
|
@Override protected void render(Block html) {
if (job == null) {
html.
p()._("Sorry, no counters for nonexistent", $(JOB_ID, "job"))._();
return;
}
if (!$(TASK_ID).isEmpty() && task == null) {
html.
p()._("Sorry, no counters for nonexistent", $(TASK_ID, "task"))._();
return;
}
if(total == null || total.getGroupNames() == null || total.countCounters() == 0) {
String type = $(TASK_ID);
if(type == null || type.isEmpty()) {
type = $(JOB_ID, "the job");
}
html.
p()._("Sorry it looks like ",type," has no counters.")._();
return;
}
String urlBase;
String urlId;
if(task != null) {
urlBase = "singletaskcounter";
urlId = MRApps.toString(task.getID());
} else {
urlBase = "singlejobcounter";
urlId = MRApps.toString(job.getID());
}
int numGroups = 0;
TBODY<TABLE<DIV<Hamlet>>> tbody = html.
div(_INFO_WRAP).
table("#counters").
thead().
tr().
th(".group.ui-state-default", "Counter Group").
th(".ui-state-default", "Counters")._()._().
tbody();
for (CounterGroup g : total) {
CounterGroup mg = map == null ? null : map.getGroup(g.getName());
CounterGroup rg = reduce == null ? null : reduce.getGroup(g.getName());
++numGroups;
// This is mostly for demonstration :) Typically we'd introduced
// a CounterGroup block to reduce the verbosity. OTOH, this
// serves as an indicator of where we're in the tag hierarchy.
TR<THEAD<TABLE<TD<TR<TBODY<TABLE<DIV<Hamlet>>>>>>>> groupHeadRow = tbody.
tr().
th().$title(g.getName()).$class("ui-state-default").
_(fixGroupDisplayName(g.getDisplayName()))._().
td().$class(C_TABLE).
table(".dt-counters").$id(job.getID()+"."+g.getName()).
thead().
tr().th(".name", "Name");
if (map != null) {
groupHeadRow.th("Map").th("Reduce");
}
// Ditto
TBODY<TABLE<TD<TR<TBODY<TABLE<DIV<Hamlet>>>>>>> group = groupHeadRow.
th(map == null ? "Value" : "Total")._()._().
tbody();
for (Counter counter : g) {
// Ditto
TR<TBODY<TABLE<TD<TR<TBODY<TABLE<DIV<Hamlet>>>>>>>> groupRow = group.
tr();
if (task == null && mg == null && rg == null) {
groupRow.td().$title(counter.getName())._(counter.getDisplayName()).
_();
} else {
groupRow.td().$title(counter.getName()).
a(url(urlBase,urlId,g.getName(),
counter.getName()), counter.getDisplayName()).
_();
}
if (map != null) {
Counter mc = mg == null ? null : mg.findCounter(counter.getName());
Counter rc = rg == null ? null : rg.findCounter(counter.getName());
groupRow.
td(mc == null ? "0" : String.format("%,d", mc.getValue())).
td(rc == null ? "0" : String.format("%,d", rc.getValue()));
}
groupRow.td(String.format("%,d", counter.getValue()))._();
}
group._()._()._()._();
}
tbody._()._()._();
}
Example 5
| Source File: CountersBlock.java From big-c with Apache License 2.0 | 4 votes |
|
@Override protected void render(Block html) {
if (job == null) {
html.
p()._("Sorry, no counters for nonexistent", $(JOB_ID, "job"))._();
return;
}
if (!$(TASK_ID).isEmpty() && task == null) {
html.
p()._("Sorry, no counters for nonexistent", $(TASK_ID, "task"))._();
return;
}
if(total == null || total.getGroupNames() == null || total.countCounters() == 0) {
String type = $(TASK_ID);
if(type == null || type.isEmpty()) {
type = $(JOB_ID, "the job");
}
html.
p()._("Sorry it looks like ",type," has no counters.")._();
return;
}
String urlBase;
String urlId;
if(task != null) {
urlBase = "singletaskcounter";
urlId = MRApps.toString(task.getID());
} else {
urlBase = "singlejobcounter";
urlId = MRApps.toString(job.getID());
}
int numGroups = 0;
TBODY<TABLE<DIV<Hamlet>>> tbody = html.
div(_INFO_WRAP).
table("#counters").
thead().
tr().
th(".group.ui-state-default", "Counter Group").
th(".ui-state-default", "Counters")._()._().
tbody();
for (CounterGroup g : total) {
CounterGroup mg = map == null ? null : map.getGroup(g.getName());
CounterGroup rg = reduce == null ? null : reduce.getGroup(g.getName());
++numGroups;
// This is mostly for demonstration :) Typically we'd introduced
// a CounterGroup block to reduce the verbosity. OTOH, this
// serves as an indicator of where we're in the tag hierarchy.
TR<THEAD<TABLE<TD<TR<TBODY<TABLE<DIV<Hamlet>>>>>>>> groupHeadRow = tbody.
tr().
th().$title(g.getName()).$class("ui-state-default").
_(fixGroupDisplayName(g.getDisplayName()))._().
td().$class(C_TABLE).
table(".dt-counters").$id(job.getID()+"."+g.getName()).
thead().
tr().th(".name", "Name");
if (map != null) {
groupHeadRow.th("Map").th("Reduce");
}
// Ditto
TBODY<TABLE<TD<TR<TBODY<TABLE<DIV<Hamlet>>>>>>> group = groupHeadRow.
th(map == null ? "Value" : "Total")._()._().
tbody();
for (Counter counter : g) {
// Ditto
TR<TBODY<TABLE<TD<TR<TBODY<TABLE<DIV<Hamlet>>>>>>>> groupRow = group.
tr();
if (task == null && mg == null && rg == null) {
groupRow.td().$title(counter.getName())._(counter.getDisplayName()).
_();
} else {
groupRow.td().$title(counter.getName()).
a(url(urlBase,urlId,g.getName(),
counter.getName()), counter.getDisplayName()).
_();
}
if (map != null) {
Counter mc = mg == null ? null : mg.findCounter(counter.getName());
Counter rc = rg == null ? null : rg.findCounter(counter.getName());
groupRow.
td(mc == null ? "0" : String.format("%,d", mc.getValue())).
td(rc == null ? "0" : String.format("%,d", rc.getValue()));
}
groupRow.td(String.format("%,d", counter.getValue()))._();
}
group._()._()._()._();
}
tbody._()._()._();
}
Example 6
| Source File: YarnJobStatsUtility.java From jumbune with GNU Lesser General Public License v3.0 | 4 votes |
|
/**
* This method is responsible for populating the reduce phase details.
* @return TaskOutputDetails contains the details of the reduce phase.
*/
private TaskOutputDetails addReducePhaseDetails(
Entry<TaskAttemptID, TaskAttemptInfo> task, long referencedZeroTime) {
TaskAttemptInfo taskAttemptInfo = (TaskAttemptInfo) (task.getValue());
TaskOutputDetails taskOutputDetails = new TaskOutputDetails();
if(taskAttemptInfo.getTaskStatus().equalsIgnoreCase("SUCCEEDED")){
taskOutputDetails.setTaskStatus(taskAttemptInfo.getTaskStatus());
taskOutputDetails.setTaskType(taskAttemptInfo.getTaskType().toString());
taskOutputDetails.setTaskID(taskAttemptInfo.getAttemptId().getTaskID().toString());
taskOutputDetails.setLocation(taskAttemptInfo.getHostname());
Counters counters = taskAttemptInfo.getCounters();
CounterGroup mapReduceTaskCounters = counters.getGroup("org.apache.hadoop.mapreduce.TaskCounter");
Counter reduceOutputRecords = mapReduceTaskCounters.findCounter("REDUCE_OUTPUT_RECORDS");
taskOutputDetails.setOutputRecords(reduceOutputRecords.getValue());
Counter reduceOutputBytes = mapReduceTaskCounters.findCounter("SPILLED_RECORDS");
taskOutputDetails.setOutputBytes(reduceOutputBytes.getValue());
long shuffleStartTime = (taskAttemptInfo.getStartTime()- referencedZeroTime)/CONVERSION_FACTOR_MILLISECS_TO_SECS;
taskOutputDetails.setStartPoint(shuffleStartTime);
taskOutputDetails.setShuffleStart(shuffleStartTime);
LOGGER.debug("shuffle start time" + taskOutputDetails.getShuffleStart());
long shuffleEnd = ((taskAttemptInfo.getShuffleFinishTime()-referencedZeroTime)/CONVERSION_FACTOR_MILLISECS_TO_SECS);
taskOutputDetails.setShuffleEnd(shuffleEnd);
LOGGER.debug("shuffle end time" + taskOutputDetails.getShuffleEnd());
taskOutputDetails.setSortStart(shuffleEnd);
long sortEnd = (taskAttemptInfo.getSortFinishTime()-referencedZeroTime)/CONVERSION_FACTOR_MILLISECS_TO_SECS;
taskOutputDetails.setSortEnd(sortEnd);
LOGGER.debug("sort end time" + taskOutputDetails.getSortEnd());
taskOutputDetails.setReduceStart(sortEnd);
taskOutputDetails.setReduceEnd((taskAttemptInfo.getFinishTime()-referencedZeroTime)/CONVERSION_FACTOR_MILLISECS_TO_SECS);
taskOutputDetails.setEndPoint(taskOutputDetails.getReduceEnd());
LOGGER.debug("Reduce end time" + taskOutputDetails.getReduceEnd());
long dataFlowRate = reduceOutputBytes.getValue() / (taskOutputDetails.getReduceEnd()-shuffleStartTime);
taskOutputDetails.setDataFlowRate(dataFlowRate);
Counter physicalMemoryBytes = mapReduceTaskCounters.findCounter("PHYSICAL_MEMORY_BYTES");
ResourceUsageMetrics rum = new ResourceUsageMetrics();
rum.setPhysicalMemoryUsage(physicalMemoryBytes.getValue());
taskOutputDetails.setResourceUsageMetrics(rum);
}
return taskOutputDetails;
}
