Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationAttemptId#toString()

The following examples show how to use org.apache.hadoop.yarn.api.records.ApplicationAttemptId#toString() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   File: ZKRMStateStore.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void updateApplicationAttemptStateInternal(
    ApplicationAttemptId appAttemptId,
    ApplicationAttemptStateData attemptStateDataPB)
    throws Exception {
  String appIdStr = appAttemptId.getApplicationId().toString();
  String appAttemptIdStr = appAttemptId.toString();
  String appDirPath = getNodePath(rmAppRoot, appIdStr);
  String nodeUpdatePath = getNodePath(appDirPath, appAttemptIdStr);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Storing final state info for attempt: " + appAttemptIdStr
        + " at: " + nodeUpdatePath);
  }
  byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();

  if (existsWithRetries(nodeUpdatePath, false) != null) {
    setDataWithRetries(nodeUpdatePath, attemptStateData, -1);
  } else {
    createWithRetries(nodeUpdatePath, attemptStateData, zkAcl,
      CreateMode.PERSISTENT);
    LOG.debug(appAttemptId + " znode didn't exist. Created a new znode to"
        + " update the application attempt state.");
  }
}
 
Example 2
Source Project: hadoop   File: TestFSRMStateStore.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void modifyAppState() throws Exception {
  // imitate appAttemptFile1 is still .new, but old one is deleted
  String appAttemptIdStr1 = "appattempt_1352994193343_0001_000001";
  ApplicationAttemptId attemptId1 =
      ConverterUtils.toApplicationAttemptId(appAttemptIdStr1);
  Path appDir =
          fsTester.store.getAppDir(attemptId1.getApplicationId().toString());
  Path appAttemptFile1 =
      new Path(appDir, attemptId1.toString() + ".new");
  FileSystemRMStateStore fileSystemRMStateStore =
      (FileSystemRMStateStore) fsTester.getRMStateStore();
  fileSystemRMStateStore.renameFile(appAttemptFile1,
          new Path(appAttemptFile1.getParent(),
                  appAttemptFile1.getName() + ".new"));
}
 
Example 3
Source Project: big-c   File: ZKRMStateStore.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void updateApplicationAttemptStateInternal(
    ApplicationAttemptId appAttemptId,
    ApplicationAttemptStateData attemptStateDataPB)
    throws Exception {
  String appIdStr = appAttemptId.getApplicationId().toString();
  String appAttemptIdStr = appAttemptId.toString();
  String appDirPath = getNodePath(rmAppRoot, appIdStr);
  String nodeUpdatePath = getNodePath(appDirPath, appAttemptIdStr);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Storing final state info for attempt: " + appAttemptIdStr
        + " at: " + nodeUpdatePath);
  }
  byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();

  if (existsWithRetries(nodeUpdatePath, false) != null) {
    setDataWithRetries(nodeUpdatePath, attemptStateData, -1);
  } else {
    createWithRetries(nodeUpdatePath, attemptStateData, zkAcl,
      CreateMode.PERSISTENT);
    LOG.debug(appAttemptId + " znode didn't exist. Created a new znode to"
        + " update the application attempt state.");
  }
}
 
Example 4
Source Project: big-c   File: TestFSRMStateStore.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void modifyAppState() throws Exception {
  // imitate appAttemptFile1 is still .new, but old one is deleted
  String appAttemptIdStr1 = "appattempt_1352994193343_0001_000001";
  ApplicationAttemptId attemptId1 =
      ConverterUtils.toApplicationAttemptId(appAttemptIdStr1);
  Path appDir =
          fsTester.store.getAppDir(attemptId1.getApplicationId().toString());
  Path appAttemptFile1 =
      new Path(appDir, attemptId1.toString() + ".new");
  FileSystemRMStateStore fileSystemRMStateStore =
      (FileSystemRMStateStore) fsTester.getRMStateStore();
  fileSystemRMStateStore.renameFile(appAttemptFile1,
          new Path(appAttemptFile1.getParent(),
                  appAttemptFile1.getName() + ".new"));
}
 
Example 5
public static void intercept(@Argument(0) String containerID, @Argument(1) long currentMemUsage,
                             @Argument(3) long limit) throws Exception {
    try {

        ContainerId cID = ConverterUtils.toContainerId(containerID);
        ApplicationAttemptId applicationAttemptId = cID.getApplicationAttemptId();
        String applicationId = applicationAttemptId.getApplicationId().toString();
        String attemptId = applicationAttemptId.toString();

        Header header = Header.newBuilder()
                .withId(applicationId)
                .withApplicationID(applicationId)
                .withAttemptID(attemptId)
                .withContainerID(containerID)
                .build();

        long memUsage = (currentMemUsage > 0) ? currentMemUsage : 0;

        ContainerEventProtos.ContainerResourceEvent event = ContainerEventProtos.ContainerResourceEvent.newBuilder()
                .setType(ContainerType.MEMORY.name())
                .setValue(memUsage)
                .setLimit(limit)
                .build();
        eventHandler.accept(System.currentTimeMillis(), header, event);
    } catch (Throwable ignored) {
    }
}
 
Example 6
public static void intercept(@This Object containerMetrics,
                             @Argument(1) int milliVcoresUsed) throws Exception {
    try {
        if (getField() != null) {
            ContainerId cID = (ContainerId) getField().get(containerMetrics);
            ApplicationAttemptId applicationAttemptId = cID.getApplicationAttemptId();
            String applicationId = applicationAttemptId.getApplicationId().toString();
            String attemptId = applicationAttemptId.toString();

            float cpuVcoreUsed = (milliVcoresUsed > 0) ? (float) milliVcoresUsed / 1000 : 0f;
            int cpuVcoreLimit = ((ContainerMetrics) containerMetrics).cpuVcoreLimit.value();

            Header header = Header.newBuilder()
                    .withId(applicationId)
                    .withApplicationID(applicationId)
                    .withAttemptID(attemptId)
                    .withContainerID(cID.toString())
                    .build();

            ContainerEventProtos.ContainerResourceEvent event = ContainerEventProtos.ContainerResourceEvent.newBuilder()
                    .setType(ContainerType.VCORE.name())
                    .setValue(cpuVcoreUsed)
                    .setLimit(cpuVcoreLimit)
                    .build();
            eventHandler.accept(System.currentTimeMillis(), header, event);
        } else {
            LOGGER.warn("ContainerMetrics class does not have containerId field");
        }
    } catch (Throwable ignored) {
    }
}
 
Example 7
Source Project: hadoop   File: NMTokenSecretManagerInNM.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This method will be used to verify NMTokens generated by different master
 * keys.
 */
@Override
public synchronized byte[] retrievePassword(NMTokenIdentifier identifier)
    throws InvalidToken {
  int keyId = identifier.getKeyId();
  ApplicationAttemptId appAttemptId = identifier.getApplicationAttemptId();

  /*
   * MasterKey used for retrieving password will be as follows. 1) By default
   * older saved master key will be used. 2) If identifier's master key id
   * matches that of previous master key id then previous key will be used. 3)
   * If identifier's master key id matches that of current master key id then
   * current key will be used.
   */
  MasterKeyData oldMasterKey = oldMasterKeys.get(appAttemptId);
  MasterKeyData masterKeyToUse = oldMasterKey;
  if (previousMasterKey != null
      && keyId == previousMasterKey.getMasterKey().getKeyId()) {
    masterKeyToUse = previousMasterKey;
  } else if (keyId == currentMasterKey.getMasterKey().getKeyId()) {
    masterKeyToUse = currentMasterKey;
  }
  
  if (nodeId != null && !identifier.getNodeId().equals(nodeId)) {
    throw new InvalidToken("Given NMToken for application : "
        + appAttemptId.toString() + " is not valid for current node manager."
        + "expected : " + nodeId.toString() + " found : "
        + identifier.getNodeId().toString());
  }
  
  if (masterKeyToUse != null) {
    byte[] password = retrivePasswordInternal(identifier, masterKeyToUse);
    LOG.debug("NMToken password retrieved successfully!!");
    return password;
  }

  throw new InvalidToken("Given NMToken for application : "
      + appAttemptId.toString() + " seems to have been generated illegally.");
}
 
Example 8
Source Project: hadoop   File: TestFSRMStateStore.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void testFSRMStateStore() throws Exception {
  HdfsConfiguration conf = new HdfsConfiguration();
  MiniDFSCluster cluster =
          new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  try {
    fsTester = new TestFSRMStateStoreTester(cluster, false);
    // If the state store is FileSystemRMStateStore then add corrupted entry.
    // It should discard the entry and remove it from file system.
    FSDataOutputStream fsOut = null;
    FileSystemRMStateStore fileSystemRMStateStore =
            (FileSystemRMStateStore) fsTester.getRMStateStore();
    String appAttemptIdStr3 = "appattempt_1352994193343_0001_000003";
    ApplicationAttemptId attemptId3 =
            ConverterUtils.toApplicationAttemptId(appAttemptIdStr3);
    Path appDir =
            fsTester.store.getAppDir(attemptId3.getApplicationId().toString());
    Path tempAppAttemptFile =
            new Path(appDir, attemptId3.toString() + ".tmp");
    fsOut = fileSystemRMStateStore.fs.create(tempAppAttemptFile, false);
    fsOut.write("Some random data ".getBytes());
    fsOut.close();

    testRMAppStateStore(fsTester);
    Assert.assertFalse(fsTester.workingDirPathURI
            .getFileSystem(conf).exists(tempAppAttemptFile));
    testRMDTSecretManagerStateStore(fsTester);
    testCheckVersion(fsTester);
    testEpoch(fsTester);
    testAppDeletion(fsTester);
    testDeleteStore(fsTester);
    testAMRMTokenSecretManagerStateStore(fsTester);
  } finally {
    cluster.shutdown();
  }
}
 
Example 9
Source Project: big-c   File: NMTokenSecretManagerInNM.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This method will be used to verify NMTokens generated by different master
 * keys.
 */
@Override
public synchronized byte[] retrievePassword(NMTokenIdentifier identifier)
    throws InvalidToken {
  int keyId = identifier.getKeyId();
  ApplicationAttemptId appAttemptId = identifier.getApplicationAttemptId();

  /*
   * MasterKey used for retrieving password will be as follows. 1) By default
   * older saved master key will be used. 2) If identifier's master key id
   * matches that of previous master key id then previous key will be used. 3)
   * If identifier's master key id matches that of current master key id then
   * current key will be used.
   */
  MasterKeyData oldMasterKey = oldMasterKeys.get(appAttemptId);
  MasterKeyData masterKeyToUse = oldMasterKey;
  if (previousMasterKey != null
      && keyId == previousMasterKey.getMasterKey().getKeyId()) {
    masterKeyToUse = previousMasterKey;
  } else if (keyId == currentMasterKey.getMasterKey().getKeyId()) {
    masterKeyToUse = currentMasterKey;
  }
  
  if (nodeId != null && !identifier.getNodeId().equals(nodeId)) {
    throw new InvalidToken("Given NMToken for application : "
        + appAttemptId.toString() + " is not valid for current node manager."
        + "expected : " + nodeId.toString() + " found : "
        + identifier.getNodeId().toString());
  }
  
  if (masterKeyToUse != null) {
    byte[] password = retrivePasswordInternal(identifier, masterKeyToUse);
    LOG.debug("NMToken password retrieved successfully!!");
    return password;
  }

  throw new InvalidToken("Given NMToken for application : "
      + appAttemptId.toString() + " seems to have been generated illegally.");
}
 
Example 10
Source Project: big-c   File: TestFSRMStateStore.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void testFSRMStateStore() throws Exception {
  HdfsConfiguration conf = new HdfsConfiguration();
  MiniDFSCluster cluster =
          new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  try {
    fsTester = new TestFSRMStateStoreTester(cluster, false);
    // If the state store is FileSystemRMStateStore then add corrupted entry.
    // It should discard the entry and remove it from file system.
    FSDataOutputStream fsOut = null;
    FileSystemRMStateStore fileSystemRMStateStore =
            (FileSystemRMStateStore) fsTester.getRMStateStore();
    String appAttemptIdStr3 = "appattempt_1352994193343_0001_000003";
    ApplicationAttemptId attemptId3 =
            ConverterUtils.toApplicationAttemptId(appAttemptIdStr3);
    Path appDir =
            fsTester.store.getAppDir(attemptId3.getApplicationId().toString());
    Path tempAppAttemptFile =
            new Path(appDir, attemptId3.toString() + ".tmp");
    fsOut = fileSystemRMStateStore.fs.create(tempAppAttemptFile, false);
    fsOut.write("Some random data ".getBytes());
    fsOut.close();

    testRMAppStateStore(fsTester);
    Assert.assertFalse(fsTester.workingDirPathURI
            .getFileSystem(conf).exists(tempAppAttemptFile));
    testRMDTSecretManagerStateStore(fsTester);
    testCheckVersion(fsTester);
    testEpoch(fsTester);
    testAppDeletion(fsTester);
    testDeleteStore(fsTester);
    testAMRMTokenSecretManagerStateStore(fsTester);
  } finally {
    cluster.shutdown();
  }
}
 
Example 11
Source Project: garmadon   File: RMContainerTracer.java    License: Apache License 2.0 4 votes vote down vote up
public static void intercept(@This Object rmContainerImpl,
                             @Argument(0) RMContainerEvent rmContainerEvent) {
    try {
        ContainerId cID = rmContainerEvent.getContainerId();
        ApplicationAttemptId applicationAttemptId = cID.getApplicationAttemptId();
        String applicationId = applicationAttemptId.getApplicationId().toString();
        String attemptId = applicationAttemptId.toString();

        Header header = Header.newBuilder()
            .withId(applicationId)
            .withApplicationID(applicationId)
            .withAttemptID(attemptId)
            .withContainerID(cID.toString())
            .build();

        RMContainerImpl rmc = (RMContainerImpl) rmContainerImpl;

        ResourceManagerEventProtos.ContainerEvent.Builder eventBuilder = ResourceManagerEventProtos.ContainerEvent.newBuilder()
            .setType(rmContainerEvent.getType().name())
            .setState(rmc.getState().name())
            .setStartTime(rmc.getCreationTime())
            .setLogUrl(rmc.getLogURL());

        if (rmc.getContainer() != null && rmc.getContainer().getResource() != null) {
            eventBuilder.setVcoresReserved(rmc.getContainer().getResource().getVirtualCores());
            eventBuilder.setMemoryReserved(rmc.getContainer().getResource().getMemory());
        }

        if (rmc.getAllocatedNode() != null) {
            eventBuilder.setContainerHostname(rmc.getAllocatedNode().getHost());
        }

        ContainerStatus containerStatus = (ContainerStatus) getField().get(rmContainerImpl);
        if (containerStatus != null) {
            eventBuilder
                .setIsFinished(true)
                .setExitStatus(rmc.getContainerExitStatus())
                .setReason(rmc.getDiagnosticsInfo())
                .setFinishTime(rmc.getFinishTime());
        }

        eventHandler.accept(System.currentTimeMillis(), header, eventBuilder.build());
    } catch (Throwable ignored) {
    }
}
 
Example 12
Source Project: hadoop   File: AMStartedEvent.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Create an event to record the start of an MR AppMaster
 *
 * @param appAttemptId
 *          the application attempt id.
 * @param startTime
 *          the start time of the AM.
 * @param containerId
 *          the containerId of the AM.
 * @param nodeManagerHost
 *          the node on which the AM is running.
 * @param nodeManagerPort
 *          the port on which the AM is running.
 * @param nodeManagerHttpPort
 *          the httpPort for the node running the AM.
 * @param forcedJobStateOnShutDown
 *          the state to force the job into
 */
public AMStartedEvent(ApplicationAttemptId appAttemptId, long startTime,
    ContainerId containerId, String nodeManagerHost, int nodeManagerPort,
    int nodeManagerHttpPort, String forcedJobStateOnShutDown,
    long submitTime) {
  datum.applicationAttemptId = new Utf8(appAttemptId.toString());
  datum.startTime = startTime;
  datum.containerId = new Utf8(containerId.toString());
  datum.nodeManagerHost = new Utf8(nodeManagerHost);
  datum.nodeManagerPort = nodeManagerPort;
  datum.nodeManagerHttpPort = nodeManagerHttpPort;
  this.forcedJobStateOnShutDown = forcedJobStateOnShutDown;
  this.submitTime = submitTime;
}
 
Example 13
Source Project: big-c   File: AMStartedEvent.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Create an event to record the start of an MR AppMaster
 *
 * @param appAttemptId
 *          the application attempt id.
 * @param startTime
 *          the start time of the AM.
 * @param containerId
 *          the containerId of the AM.
 * @param nodeManagerHost
 *          the node on which the AM is running.
 * @param nodeManagerPort
 *          the port on which the AM is running.
 * @param nodeManagerHttpPort
 *          the httpPort for the node running the AM.
 * @param forcedJobStateOnShutDown
 *          the state to force the job into
 */
public AMStartedEvent(ApplicationAttemptId appAttemptId, long startTime,
    ContainerId containerId, String nodeManagerHost, int nodeManagerPort,
    int nodeManagerHttpPort, String forcedJobStateOnShutDown,
    long submitTime) {
  datum.applicationAttemptId = new Utf8(appAttemptId.toString());
  datum.startTime = startTime;
  datum.containerId = new Utf8(containerId.toString());
  datum.nodeManagerHost = new Utf8(nodeManagerHost);
  datum.nodeManagerPort = nodeManagerPort;
  datum.nodeManagerHttpPort = nodeManagerHttpPort;
  this.forcedJobStateOnShutDown = forcedJobStateOnShutDown;
  this.submitTime = submitTime;
}