org.apache.hadoop.yarn.api.records.YarnClusterMetrics Java Examples
The following examples show how to use
org.apache.hadoop.yarn.api.records.YarnClusterMetrics.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: GetClusterMetricsResponsePBImpl.java From big-c with Apache License 2.0 | 5 votes |
|
@Override
public void setClusterMetrics(YarnClusterMetrics clusterMetrics) {
maybeInitBuilder();
if (clusterMetrics == null)
builder.clearClusterMetrics();
this.yarnClusterMetrics = clusterMetrics;
}
Example #2
| Source File: ResourceMgrDelegate.java From tez with Apache License 2.0 | 5 votes |
|
public ClusterMetrics getClusterMetrics() throws IOException,
InterruptedException {
YarnClusterMetrics metrics;
try {
metrics = client.getYarnClusterMetrics();
} catch (YarnException e) {
throw new IOException(e);
}
ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1,
metrics.getNumNodeManagers() * 10, metrics.getNumNodeManagers() * 2, 1,
metrics.getNumNodeManagers(), 0, 0);
return oldMetrics;
}
Example #3
| Source File: Client.java From stratosphere with Apache License 2.0 | 5 votes |
|
private void showClusterMetrics(YarnClient yarnClient)
throws YarnException, IOException {
YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();
System.out.println("NodeManagers in Cluster " + metrics.getNumNodeManagers());
List<NodeReport> nodes = yarnClient.getNodeReports();
final String format = "|%-16s |%-16s %n";
System.out.printf("|Property |Value %n");
System.out.println("+---------------------------------------+");
int totalMemory = 0;
int totalCores = 0;
for(NodeReport rep : nodes) {
final Resource res = rep.getCapability();
totalMemory += res.getMemory();
totalCores += res.getVirtualCores();
System.out.format(format, "NodeID", rep.getNodeId());
System.out.format(format, "Memory", res.getMemory()+" MB");
System.out.format(format, "vCores", res.getVirtualCores());
System.out.format(format, "HealthReport", rep.getHealthReport());
System.out.format(format, "Containers", rep.getNumContainers());
System.out.println("+---------------------------------------+");
}
System.out.println("Summary: totalMemory "+totalMemory+" totalCores "+totalCores);
List<QueueInfo> qInfo = yarnClient.getAllQueues();
for(QueueInfo q : qInfo) {
System.out.println("Queue: "+q.getQueueName()+", Current Capacity: "+q.getCurrentCapacity()+" Max Capacity: "+q.getMaximumCapacity()+" Applications: "+q.getApplications().size());
}
yarnClient.stop();
System.exit(0);
}
Example #4
| Source File: ResourceMgrDelegate.java From incubator-tez with Apache License 2.0 | 5 votes |
|
public ClusterMetrics getClusterMetrics() throws IOException,
InterruptedException {
YarnClusterMetrics metrics;
try {
metrics = client.getYarnClusterMetrics();
} catch (YarnException e) {
throw new IOException(e);
}
ClusterMetrics oldMetrics = new ClusterMetrics(1, 1, 1, 1, 1, 1,
metrics.getNumNodeManagers() * 10, metrics.getNumNodeManagers() * 2, 1,
metrics.getNumNodeManagers(), 0, 0);
return oldMetrics;
}
Example #5
| Source File: YarnClusterDescriptor.java From flink with Apache License 2.0 | 5 votes |
|
@Override
public String getClusterDescription() {
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(baos);
YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();
ps.append("NodeManagers in the ClusterClient " + metrics.getNumNodeManagers());
List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);
final String format = "|%-16s |%-16s %n";
ps.printf("|Property |Value %n");
ps.println("+---------------------------------------+");
int totalMemory = 0;
int totalCores = 0;
for (NodeReport rep : nodes) {
final Resource res = rep.getCapability();
totalMemory += res.getMemory();
totalCores += res.getVirtualCores();
ps.format(format, "NodeID", rep.getNodeId());
ps.format(format, "Memory", res.getMemory() + " MB");
ps.format(format, "vCores", res.getVirtualCores());
ps.format(format, "HealthReport", rep.getHealthReport());
ps.format(format, "Containers", rep.getNumContainers());
ps.println("+---------------------------------------+");
}
ps.println("Summary: totalMemory " + totalMemory + " totalCores " + totalCores);
List<QueueInfo> qInfo = yarnClient.getAllQueues();
for (QueueInfo q : qInfo) {
ps.println("Queue: " + q.getQueueName() + ", Current Capacity: " + q.getCurrentCapacity() + " Max Capacity: " +
q.getMaximumCapacity() + " Applications: " + q.getApplications().size());
}
return baos.toString();
} catch (Exception e) {
throw new RuntimeException("Couldn't get cluster description", e);
}
}
Example #6
| Source File: ResourceMgrDelegate.java From big-c with Apache License 2.0 | 5 votes |
|
public ClusterMetrics getClusterMetrics() throws IOException,
InterruptedException {
try {
YarnClusterMetrics metrics = client.getYarnClusterMetrics();
ClusterMetrics oldMetrics =
new ClusterMetrics(1, 1, 1, 1, 1, 1,
metrics.getNumNodeManagers() * 10,
metrics.getNumNodeManagers() * 2, 1,
metrics.getNumNodeManagers(), 0, 0);
return oldMetrics;
} catch (YarnException e) {
throw new IOException(e);
}
}
Example #7
| Source File: GetClusterMetricsResponse.java From big-c with Apache License 2.0 | 5 votes |
|
@Private
@Unstable
public static GetClusterMetricsResponse
newInstance(YarnClusterMetrics metrics) {
GetClusterMetricsResponse response =
Records.newRecord(GetClusterMetricsResponse.class);
response.setClusterMetrics(metrics);
return response;
}
Example #8
| Source File: TestApplicationClientProtocolOnHA.java From big-c with Apache License 2.0 | 5 votes |
|
@Test(timeout = 15000)
public void testGetClusterMetricsOnHA() throws Exception {
YarnClusterMetrics clusterMetrics =
client.getYarnClusterMetrics();
Assert.assertTrue(clusterMetrics != null);
Assert.assertEquals(cluster.createFakeYarnClusterMetrics(),
clusterMetrics);
}
Example #9
| Source File: YarnClientImpl.java From big-c with Apache License 2.0 | 5 votes |
|
@Override
public YarnClusterMetrics getYarnClusterMetrics() throws YarnException,
IOException {
GetClusterMetricsRequest request =
Records.newRecord(GetClusterMetricsRequest.class);
GetClusterMetricsResponse response = rmClient.getClusterMetrics(request);
return response.getClusterMetrics();
}
Example #10
| Source File: ClientRMService.java From big-c with Apache License 2.0 | 5 votes |
|
@Override
public GetClusterMetricsResponse getClusterMetrics(
GetClusterMetricsRequest request) throws YarnException {
GetClusterMetricsResponse response = recordFactory
.newRecordInstance(GetClusterMetricsResponse.class);
YarnClusterMetrics ymetrics = recordFactory
.newRecordInstance(YarnClusterMetrics.class);
ymetrics.setNumNodeManagers(this.rmContext.getRMNodes().size());
response.setClusterMetrics(ymetrics);
return response;
}
Example #11
| Source File: AbstractYarnClusterDescriptor.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
@Override
public String getClusterDescription() {
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(baos);
YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();
ps.append("NodeManagers in the ClusterClient " + metrics.getNumNodeManagers());
List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);
final String format = "|%-16s |%-16s %n";
ps.printf("|Property |Value %n");
ps.println("+---------------------------------------+");
int totalMemory = 0;
int totalCores = 0;
for (NodeReport rep : nodes) {
final Resource res = rep.getCapability();
totalMemory += res.getMemory();
totalCores += res.getVirtualCores();
ps.format(format, "NodeID", rep.getNodeId());
ps.format(format, "Memory", res.getMemory() + " MB");
ps.format(format, "vCores", res.getVirtualCores());
ps.format(format, "HealthReport", rep.getHealthReport());
ps.format(format, "Containers", rep.getNumContainers());
ps.println("+---------------------------------------+");
}
ps.println("Summary: totalMemory " + totalMemory + " totalCores " + totalCores);
List<QueueInfo> qInfo = yarnClient.getAllQueues();
for (QueueInfo q : qInfo) {
ps.println("Queue: " + q.getQueueName() + ", Current Capacity: " + q.getCurrentCapacity() + " Max Capacity: " +
q.getMaximumCapacity() + " Applications: " + q.getApplications().size());
}
return baos.toString();
} catch (Exception e) {
throw new RuntimeException("Couldn't get cluster description", e);
}
}
Example #12
| Source File: GetClusterMetricsResponsePBImpl.java From big-c with Apache License 2.0 | 5 votes |
|
@Override
public YarnClusterMetrics getClusterMetrics() {
GetClusterMetricsResponseProtoOrBuilder p = viaProto ? proto : builder;
if (this.yarnClusterMetrics != null) {
return this.yarnClusterMetrics;
}
if (!p.hasClusterMetrics()) {
return null;
}
this.yarnClusterMetrics = convertFromProtoFormat(p.getClusterMetrics());
return this.yarnClusterMetrics;
}
Example #13
| Source File: ResourceMgrDelegate.java From hadoop with Apache License 2.0 | 5 votes |
|
public ClusterMetrics getClusterMetrics() throws IOException,
InterruptedException {
try {
YarnClusterMetrics metrics = client.getYarnClusterMetrics();
ClusterMetrics oldMetrics =
new ClusterMetrics(1, 1, 1, 1, 1, 1,
metrics.getNumNodeManagers() * 10,
metrics.getNumNodeManagers() * 2, 1,
metrics.getNumNodeManagers(), 0, 0);
return oldMetrics;
} catch (YarnException e) {
throw new IOException(e);
}
}
Example #14
| Source File: AbstractYarnClusterDescriptor.java From flink with Apache License 2.0 | 5 votes |
|
@Override
public String getClusterDescription() {
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(baos);
YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();
ps.append("NodeManagers in the ClusterClient " + metrics.getNumNodeManagers());
List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);
final String format = "|%-16s |%-16s %n";
ps.printf("|Property |Value %n");
ps.println("+---------------------------------------+");
int totalMemory = 0;
int totalCores = 0;
for (NodeReport rep : nodes) {
final Resource res = rep.getCapability();
totalMemory += res.getMemory();
totalCores += res.getVirtualCores();
ps.format(format, "NodeID", rep.getNodeId());
ps.format(format, "Memory", res.getMemory() + " MB");
ps.format(format, "vCores", res.getVirtualCores());
ps.format(format, "HealthReport", rep.getHealthReport());
ps.format(format, "Containers", rep.getNumContainers());
ps.println("+---------------------------------------+");
}
ps.println("Summary: totalMemory " + totalMemory + " totalCores " + totalCores);
List<QueueInfo> qInfo = yarnClient.getAllQueues();
for (QueueInfo q : qInfo) {
ps.println("Queue: " + q.getQueueName() + ", Current Capacity: " + q.getCurrentCapacity() + " Max Capacity: " +
q.getMaximumCapacity() + " Applications: " + q.getApplications().size());
}
return baos.toString();
} catch (Exception e) {
throw new RuntimeException("Couldn't get cluster description", e);
}
}
Example #15
| Source File: GetClusterMetricsResponsePBImpl.java From hadoop with Apache License 2.0 | 5 votes |
|
@Override
public YarnClusterMetrics getClusterMetrics() {
GetClusterMetricsResponseProtoOrBuilder p = viaProto ? proto : builder;
if (this.yarnClusterMetrics != null) {
return this.yarnClusterMetrics;
}
if (!p.hasClusterMetrics()) {
return null;
}
this.yarnClusterMetrics = convertFromProtoFormat(p.getClusterMetrics());
return this.yarnClusterMetrics;
}
Example #16
| Source File: GetClusterMetricsResponsePBImpl.java From hadoop with Apache License 2.0 | 5 votes |
|
@Override
public void setClusterMetrics(YarnClusterMetrics clusterMetrics) {
maybeInitBuilder();
if (clusterMetrics == null)
builder.clearClusterMetrics();
this.yarnClusterMetrics = clusterMetrics;
}
Example #17
| Source File: GetClusterMetricsResponse.java From hadoop with Apache License 2.0 | 5 votes |
|
@Private
@Unstable
public static GetClusterMetricsResponse
newInstance(YarnClusterMetrics metrics) {
GetClusterMetricsResponse response =
Records.newRecord(GetClusterMetricsResponse.class);
response.setClusterMetrics(metrics);
return response;
}
Example #18
| Source File: TestApplicationClientProtocolOnHA.java From hadoop with Apache License 2.0 | 5 votes |
|
@Test(timeout = 15000)
public void testGetClusterMetricsOnHA() throws Exception {
YarnClusterMetrics clusterMetrics =
client.getYarnClusterMetrics();
Assert.assertTrue(clusterMetrics != null);
Assert.assertEquals(cluster.createFakeYarnClusterMetrics(),
clusterMetrics);
}
Example #19
| Source File: YarnClientImpl.java From hadoop with Apache License 2.0 | 5 votes |
|
@Override
public YarnClusterMetrics getYarnClusterMetrics() throws YarnException,
IOException {
GetClusterMetricsRequest request =
Records.newRecord(GetClusterMetricsRequest.class);
GetClusterMetricsResponse response = rmClient.getClusterMetrics(request);
return response.getClusterMetrics();
}
Example #20
| Source File: ClientRMService.java From hadoop with Apache License 2.0 | 5 votes |
|
@Override
public GetClusterMetricsResponse getClusterMetrics(
GetClusterMetricsRequest request) throws YarnException {
GetClusterMetricsResponse response = recordFactory
.newRecordInstance(GetClusterMetricsResponse.class);
YarnClusterMetrics ymetrics = recordFactory
.newRecordInstance(YarnClusterMetrics.class);
ymetrics.setNumNodeManagers(this.rmContext.getRMNodes().size());
response.setClusterMetrics(ymetrics);
return response;
}
Example #21
| Source File: ProtocolHATestBase.java From big-c with Apache License 2.0 | 4 votes |
|
public YarnClusterMetrics createFakeYarnClusterMetrics() {
return YarnClusterMetrics.newInstance(1);
}
Example #22
| Source File: GetClusterMetricsResponsePBImpl.java From hadoop with Apache License 2.0 | 4 votes |
|
private YarnClusterMetricsProto convertToProtoFormat(YarnClusterMetrics t) {
return ((YarnClusterMetricsPBImpl)t).getProto();
}
Example #23
| Source File: TestYARNRunner.java From big-c with Apache License 2.0 | 4 votes |
|
@Test(timeout=20000)
public void testResourceMgrDelegate() throws Exception {
/* we not want a mock of resource mgr delegate */
final ApplicationClientProtocol clientRMProtocol = mock(ApplicationClientProtocol.class);
ResourceMgrDelegate delegate = new ResourceMgrDelegate(conf) {
@Override
protected void serviceStart() throws Exception {
assertTrue(this.client instanceof YarnClientImpl);
((YarnClientImpl) this.client).setRMClient(clientRMProtocol);
}
};
/* make sure kill calls finish application master */
when(clientRMProtocol.forceKillApplication(any(KillApplicationRequest.class)))
.thenReturn(KillApplicationResponse.newInstance(true));
delegate.killApplication(appId);
verify(clientRMProtocol).forceKillApplication(any(KillApplicationRequest.class));
/* make sure getalljobs calls get all applications */
when(clientRMProtocol.getApplications(any(GetApplicationsRequest.class))).
thenReturn(recordFactory.newRecordInstance(GetApplicationsResponse.class));
delegate.getAllJobs();
verify(clientRMProtocol).getApplications(any(GetApplicationsRequest.class));
/* make sure getapplication report is called */
when(clientRMProtocol.getApplicationReport(any(GetApplicationReportRequest.class)))
.thenReturn(recordFactory.newRecordInstance(GetApplicationReportResponse.class));
delegate.getApplicationReport(appId);
verify(clientRMProtocol).getApplicationReport(any(GetApplicationReportRequest.class));
/* make sure metrics is called */
GetClusterMetricsResponse clusterMetricsResponse = recordFactory.newRecordInstance
(GetClusterMetricsResponse.class);
clusterMetricsResponse.setClusterMetrics(recordFactory.newRecordInstance(
YarnClusterMetrics.class));
when(clientRMProtocol.getClusterMetrics(any(GetClusterMetricsRequest.class)))
.thenReturn(clusterMetricsResponse);
delegate.getClusterMetrics();
verify(clientRMProtocol).getClusterMetrics(any(GetClusterMetricsRequest.class));
when(clientRMProtocol.getClusterNodes(any(GetClusterNodesRequest.class))).
thenReturn(recordFactory.newRecordInstance(GetClusterNodesResponse.class));
delegate.getActiveTrackers();
verify(clientRMProtocol).getClusterNodes(any(GetClusterNodesRequest.class));
GetNewApplicationResponse newAppResponse = recordFactory.newRecordInstance(
GetNewApplicationResponse.class);
newAppResponse.setApplicationId(appId);
when(clientRMProtocol.getNewApplication(any(GetNewApplicationRequest.class))).
thenReturn(newAppResponse);
delegate.getNewJobID();
verify(clientRMProtocol).getNewApplication(any(GetNewApplicationRequest.class));
GetQueueInfoResponse queueInfoResponse = recordFactory.newRecordInstance(
GetQueueInfoResponse.class);
queueInfoResponse.setQueueInfo(recordFactory.newRecordInstance(QueueInfo.class));
when(clientRMProtocol.getQueueInfo(any(GetQueueInfoRequest.class))).
thenReturn(queueInfoResponse);
delegate.getQueues();
verify(clientRMProtocol).getQueueInfo(any(GetQueueInfoRequest.class));
GetQueueUserAclsInfoResponse aclResponse = recordFactory.newRecordInstance(
GetQueueUserAclsInfoResponse.class);
when(clientRMProtocol.getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class)))
.thenReturn(aclResponse);
delegate.getQueueAclsForCurrentUser();
verify(clientRMProtocol).getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class));
}
Example #24
| Source File: ResourceMgrDelegate.java From big-c with Apache License 2.0 | 4 votes |
|
@Override
public YarnClusterMetrics getYarnClusterMetrics() throws YarnException,
IOException {
return client.getYarnClusterMetrics();
}
Example #25
| Source File: GetClusterMetricsResponse.java From big-c with Apache License 2.0 | 4 votes |
|
@Private @Unstable public abstract void setClusterMetrics(YarnClusterMetrics metrics);
Example #26
| Source File: GetClusterMetricsResponse.java From hadoop with Apache License 2.0 | 4 votes |
|
@Private @Unstable public abstract void setClusterMetrics(YarnClusterMetrics metrics);
Example #27
| Source File: ProtocolHATestBase.java From hadoop with Apache License 2.0 | 4 votes |
|
public YarnClusterMetrics createFakeYarnClusterMetrics() {
return YarnClusterMetrics.newInstance(1);
}
Example #28
| Source File: GetClusterMetricsResponsePBImpl.java From big-c with Apache License 2.0 | 4 votes |
|
private YarnClusterMetricsProto convertToProtoFormat(YarnClusterMetrics t) {
return ((YarnClusterMetricsPBImpl)t).getProto();
}
Example #29
| Source File: TestYARNRunner.java From hadoop with Apache License 2.0 | 4 votes |
|
@Test(timeout=20000)
public void testResourceMgrDelegate() throws Exception {
/* we not want a mock of resource mgr delegate */
final ApplicationClientProtocol clientRMProtocol = mock(ApplicationClientProtocol.class);
ResourceMgrDelegate delegate = new ResourceMgrDelegate(conf) {
@Override
protected void serviceStart() throws Exception {
assertTrue(this.client instanceof YarnClientImpl);
((YarnClientImpl) this.client).setRMClient(clientRMProtocol);
}
};
/* make sure kill calls finish application master */
when(clientRMProtocol.forceKillApplication(any(KillApplicationRequest.class)))
.thenReturn(KillApplicationResponse.newInstance(true));
delegate.killApplication(appId);
verify(clientRMProtocol).forceKillApplication(any(KillApplicationRequest.class));
/* make sure getalljobs calls get all applications */
when(clientRMProtocol.getApplications(any(GetApplicationsRequest.class))).
thenReturn(recordFactory.newRecordInstance(GetApplicationsResponse.class));
delegate.getAllJobs();
verify(clientRMProtocol).getApplications(any(GetApplicationsRequest.class));
/* make sure getapplication report is called */
when(clientRMProtocol.getApplicationReport(any(GetApplicationReportRequest.class)))
.thenReturn(recordFactory.newRecordInstance(GetApplicationReportResponse.class));
delegate.getApplicationReport(appId);
verify(clientRMProtocol).getApplicationReport(any(GetApplicationReportRequest.class));
/* make sure metrics is called */
GetClusterMetricsResponse clusterMetricsResponse = recordFactory.newRecordInstance
(GetClusterMetricsResponse.class);
clusterMetricsResponse.setClusterMetrics(recordFactory.newRecordInstance(
YarnClusterMetrics.class));
when(clientRMProtocol.getClusterMetrics(any(GetClusterMetricsRequest.class)))
.thenReturn(clusterMetricsResponse);
delegate.getClusterMetrics();
verify(clientRMProtocol).getClusterMetrics(any(GetClusterMetricsRequest.class));
when(clientRMProtocol.getClusterNodes(any(GetClusterNodesRequest.class))).
thenReturn(recordFactory.newRecordInstance(GetClusterNodesResponse.class));
delegate.getActiveTrackers();
verify(clientRMProtocol).getClusterNodes(any(GetClusterNodesRequest.class));
GetNewApplicationResponse newAppResponse = recordFactory.newRecordInstance(
GetNewApplicationResponse.class);
newAppResponse.setApplicationId(appId);
when(clientRMProtocol.getNewApplication(any(GetNewApplicationRequest.class))).
thenReturn(newAppResponse);
delegate.getNewJobID();
verify(clientRMProtocol).getNewApplication(any(GetNewApplicationRequest.class));
GetQueueInfoResponse queueInfoResponse = recordFactory.newRecordInstance(
GetQueueInfoResponse.class);
queueInfoResponse.setQueueInfo(recordFactory.newRecordInstance(QueueInfo.class));
when(clientRMProtocol.getQueueInfo(any(GetQueueInfoRequest.class))).
thenReturn(queueInfoResponse);
delegate.getQueues();
verify(clientRMProtocol).getQueueInfo(any(GetQueueInfoRequest.class));
GetQueueUserAclsInfoResponse aclResponse = recordFactory.newRecordInstance(
GetQueueUserAclsInfoResponse.class);
when(clientRMProtocol.getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class)))
.thenReturn(aclResponse);
delegate.getQueueAclsForCurrentUser();
verify(clientRMProtocol).getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class));
}
Example #30
| Source File: ResourceMgrDelegate.java From hadoop with Apache License 2.0 | 4 votes |
|
@Override
public YarnClusterMetrics getYarnClusterMetrics() throws YarnException,
IOException {
return client.getYarnClusterMetrics();
}
