Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationAttemptId#toString()

@Override
public synchronized void updateApplicationAttemptStateInternal(
    ApplicationAttemptId appAttemptId,
    ApplicationAttemptStateData attemptStateDataPB)
    throws Exception {
  String appIdStr = appAttemptId.getApplicationId().toString();
  String appAttemptIdStr = appAttemptId.toString();
  String appDirPath = getNodePath(rmAppRoot, appIdStr);
  String nodeUpdatePath = getNodePath(appDirPath, appAttemptIdStr);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Storing final state info for attempt: " + appAttemptIdStr
        + " at: " + nodeUpdatePath);
  }
  byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();

  if (existsWithRetries(nodeUpdatePath, false) != null) {
    setDataWithRetries(nodeUpdatePath, attemptStateData, -1);
  } else {
    createWithRetries(nodeUpdatePath, attemptStateData, zkAcl,
      CreateMode.PERSISTENT);
    LOG.debug(appAttemptId + " znode didn't exist. Created a new znode to"
        + " update the application attempt state.");
  }
}
 

@Override
protected void modifyAppState() throws Exception {
  // imitate appAttemptFile1 is still .new, but old one is deleted
  String appAttemptIdStr1 = "appattempt_1352994193343_0001_000001";
  ApplicationAttemptId attemptId1 =
      ConverterUtils.toApplicationAttemptId(appAttemptIdStr1);
  Path appDir =
          fsTester.store.getAppDir(attemptId1.getApplicationId().toString());
  Path appAttemptFile1 =
      new Path(appDir, attemptId1.toString() + ".new");
  FileSystemRMStateStore fileSystemRMStateStore =
      (FileSystemRMStateStore) fsTester.getRMStateStore();
  fileSystemRMStateStore.renameFile(appAttemptFile1,
          new Path(appAttemptFile1.getParent(),
                  appAttemptFile1.getName() + ".new"));
}
 

@Override
public synchronized void updateApplicationAttemptStateInternal(
    ApplicationAttemptId appAttemptId,
    ApplicationAttemptStateData attemptStateDataPB)
    throws Exception {
  String appIdStr = appAttemptId.getApplicationId().toString();
  String appAttemptIdStr = appAttemptId.toString();
  String appDirPath = getNodePath(rmAppRoot, appIdStr);
  String nodeUpdatePath = getNodePath(appDirPath, appAttemptIdStr);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Storing final state info for attempt: " + appAttemptIdStr
        + " at: " + nodeUpdatePath);
  }
  byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();

  if (existsWithRetries(nodeUpdatePath, false) != null) {
    setDataWithRetries(nodeUpdatePath, attemptStateData, -1);
  } else {
    createWithRetries(nodeUpdatePath, attemptStateData, zkAcl,
      CreateMode.PERSISTENT);
    LOG.debug(appAttemptId + " znode didn't exist. Created a new znode to"
        + " update the application attempt state.");
  }
}
 

@Override
protected void modifyAppState() throws Exception {
  // imitate appAttemptFile1 is still .new, but old one is deleted
  String appAttemptIdStr1 = "appattempt_1352994193343_0001_000001";
  ApplicationAttemptId attemptId1 =
      ConverterUtils.toApplicationAttemptId(appAttemptIdStr1);
  Path appDir =
          fsTester.store.getAppDir(attemptId1.getApplicationId().toString());
  Path appAttemptFile1 =
      new Path(appDir, attemptId1.toString() + ".new");
  FileSystemRMStateStore fileSystemRMStateStore =
      (FileSystemRMStateStore) fsTester.getRMStateStore();
  fileSystemRMStateStore.renameFile(appAttemptFile1,
          new Path(appAttemptFile1.getParent(),
                  appAttemptFile1.getName() + ".new"));
}
 

public static void intercept(@Argument(0) String containerID, @Argument(1) long currentMemUsage,
                             @Argument(3) long limit) throws Exception {
    try {

        ContainerId cID = ConverterUtils.toContainerId(containerID);
        ApplicationAttemptId applicationAttemptId = cID.getApplicationAttemptId();
        String applicationId = applicationAttemptId.getApplicationId().toString();
        String attemptId = applicationAttemptId.toString();

        Header header = Header.newBuilder()
                .withId(applicationId)
                .withApplicationID(applicationId)
                .withAttemptID(attemptId)
                .withContainerID(containerID)
                .build();

        long memUsage = (currentMemUsage > 0) ? currentMemUsage : 0;

        ContainerEventProtos.ContainerResourceEvent event = ContainerEventProtos.ContainerResourceEvent.newBuilder()
                .setType(ContainerType.MEMORY.name())
                .setValue(memUsage)
                .setLimit(limit)
                .build();
        eventHandler.accept(System.currentTimeMillis(), header, event);
    } catch (Throwable ignored) {
    }
}
 

public static void intercept(@This Object containerMetrics,
                             @Argument(1) int milliVcoresUsed) throws Exception {
    try {
        if (getField() != null) {
            ContainerId cID = (ContainerId) getField().get(containerMetrics);
            ApplicationAttemptId applicationAttemptId = cID.getApplicationAttemptId();
            String applicationId = applicationAttemptId.getApplicationId().toString();
            String attemptId = applicationAttemptId.toString();

            float cpuVcoreUsed = (milliVcoresUsed > 0) ? (float) milliVcoresUsed / 1000 : 0f;
            int cpuVcoreLimit = ((ContainerMetrics) containerMetrics).cpuVcoreLimit.value();

            Header header = Header.newBuilder()
                    .withId(applicationId)
                    .withApplicationID(applicationId)
                    .withAttemptID(attemptId)
                    .withContainerID(cID.toString())
                    .build();

            ContainerEventProtos.ContainerResourceEvent event = ContainerEventProtos.ContainerResourceEvent.newBuilder()
                    .setType(ContainerType.VCORE.name())
                    .setValue(cpuVcoreUsed)
                    .setLimit(cpuVcoreLimit)
                    .build();
            eventHandler.accept(System.currentTimeMillis(), header, event);
        } else {
            LOGGER.warn("ContainerMetrics class does not have containerId field");
        }
    } catch (Throwable ignored) {
    }
}
 

/**
 * This method will be used to verify NMTokens generated by different master
 * keys.
 */
@Override
public synchronized byte[] retrievePassword(NMTokenIdentifier identifier)
    throws InvalidToken {
  int keyId = identifier.getKeyId();
  ApplicationAttemptId appAttemptId = identifier.getApplicationAttemptId();

  /*
   * MasterKey used for retrieving password will be as follows. 1) By default
   * older saved master key will be used. 2) If identifier's master key id
   * matches that of previous master key id then previous key will be used. 3)
   * If identifier's master key id matches that of current master key id then
   * current key will be used.
   */
  MasterKeyData oldMasterKey = oldMasterKeys.get(appAttemptId);
  MasterKeyData masterKeyToUse = oldMasterKey;
  if (previousMasterKey != null
      && keyId == previousMasterKey.getMasterKey().getKeyId()) {
    masterKeyToUse = previousMasterKey;
  } else if (keyId == currentMasterKey.getMasterKey().getKeyId()) {
    masterKeyToUse = currentMasterKey;
  }
  
  if (nodeId != null && !identifier.getNodeId().equals(nodeId)) {
    throw new InvalidToken("Given NMToken for application : "
        + appAttemptId.toString() + " is not valid for current node manager."
        + "expected : " + nodeId.toString() + " found : "
        + identifier.getNodeId().toString());
  }
  
  if (masterKeyToUse != null) {
    byte[] password = retrivePasswordInternal(identifier, masterKeyToUse);
    LOG.debug("NMToken password retrieved successfully!!");
    return password;
  }

  throw new InvalidToken("Given NMToken for application : "
      + appAttemptId.toString() + " seems to have been generated illegally.");
}
 

@Test(timeout = 60000)
public void testFSRMStateStore() throws Exception {
  HdfsConfiguration conf = new HdfsConfiguration();
  MiniDFSCluster cluster =
          new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  try {
    fsTester = new TestFSRMStateStoreTester(cluster, false);
    // If the state store is FileSystemRMStateStore then add corrupted entry.
    // It should discard the entry and remove it from file system.
    FSDataOutputStream fsOut = null;
    FileSystemRMStateStore fileSystemRMStateStore =
            (FileSystemRMStateStore) fsTester.getRMStateStore();
    String appAttemptIdStr3 = "appattempt_1352994193343_0001_000003";
    ApplicationAttemptId attemptId3 =
            ConverterUtils.toApplicationAttemptId(appAttemptIdStr3);
    Path appDir =
            fsTester.store.getAppDir(attemptId3.getApplicationId().toString());
    Path tempAppAttemptFile =
            new Path(appDir, attemptId3.toString() + ".tmp");
    fsOut = fileSystemRMStateStore.fs.create(tempAppAttemptFile, false);
    fsOut.write("Some random data ".getBytes());
    fsOut.close();

    testRMAppStateStore(fsTester);
    Assert.assertFalse(fsTester.workingDirPathURI
            .getFileSystem(conf).exists(tempAppAttemptFile));
    testRMDTSecretManagerStateStore(fsTester);
    testCheckVersion(fsTester);
    testEpoch(fsTester);
    testAppDeletion(fsTester);
    testDeleteStore(fsTester);
    testAMRMTokenSecretManagerStateStore(fsTester);
  } finally {
    cluster.shutdown();
  }
}
 

/**
 * This method will be used to verify NMTokens generated by different master
 * keys.
 */
@Override
public synchronized byte[] retrievePassword(NMTokenIdentifier identifier)
    throws InvalidToken {
  int keyId = identifier.getKeyId();
  ApplicationAttemptId appAttemptId = identifier.getApplicationAttemptId();

  /*
   * MasterKey used for retrieving password will be as follows. 1) By default
   * older saved master key will be used. 2) If identifier's master key id
   * matches that of previous master key id then previous key will be used. 3)
   * If identifier's master key id matches that of current master key id then
   * current key will be used.
   */
  MasterKeyData oldMasterKey = oldMasterKeys.get(appAttemptId);
  MasterKeyData masterKeyToUse = oldMasterKey;
  if (previousMasterKey != null
      && keyId == previousMasterKey.getMasterKey().getKeyId()) {
    masterKeyToUse = previousMasterKey;
  } else if (keyId == currentMasterKey.getMasterKey().getKeyId()) {
    masterKeyToUse = currentMasterKey;
  }
  
  if (nodeId != null && !identifier.getNodeId().equals(nodeId)) {
    throw new InvalidToken("Given NMToken for application : "
        + appAttemptId.toString() + " is not valid for current node manager."
        + "expected : " + nodeId.toString() + " found : "
        + identifier.getNodeId().toString());
  }
  
  if (masterKeyToUse != null) {
    byte[] password = retrivePasswordInternal(identifier, masterKeyToUse);
    LOG.debug("NMToken password retrieved successfully!!");
    return password;
  }

  throw new InvalidToken("Given NMToken for application : "
      + appAttemptId.toString() + " seems to have been generated illegally.");
}
 

@Test(timeout = 60000)
public void testFSRMStateStore() throws Exception {
  HdfsConfiguration conf = new HdfsConfiguration();
  MiniDFSCluster cluster =
          new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  try {
    fsTester = new TestFSRMStateStoreTester(cluster, false);
    // If the state store is FileSystemRMStateStore then add corrupted entry.
    // It should discard the entry and remove it from file system.
    FSDataOutputStream fsOut = null;
    FileSystemRMStateStore fileSystemRMStateStore =
            (FileSystemRMStateStore) fsTester.getRMStateStore();
    String appAttemptIdStr3 = "appattempt_1352994193343_0001_000003";
    ApplicationAttemptId attemptId3 =
            ConverterUtils.toApplicationAttemptId(appAttemptIdStr3);
    Path appDir =
            fsTester.store.getAppDir(attemptId3.getApplicationId().toString());
    Path tempAppAttemptFile =
            new Path(appDir, attemptId3.toString() + ".tmp");
    fsOut = fileSystemRMStateStore.fs.create(tempAppAttemptFile, false);
    fsOut.write("Some random data ".getBytes());
    fsOut.close();

    testRMAppStateStore(fsTester);
    Assert.assertFalse(fsTester.workingDirPathURI
            .getFileSystem(conf).exists(tempAppAttemptFile));
    testRMDTSecretManagerStateStore(fsTester);
    testCheckVersion(fsTester);
    testEpoch(fsTester);
    testAppDeletion(fsTester);
    testDeleteStore(fsTester);
    testAMRMTokenSecretManagerStateStore(fsTester);
  } finally {
    cluster.shutdown();
  }
}
 

public static void intercept(@This Object rmContainerImpl,
                             @Argument(0) RMContainerEvent rmContainerEvent) {
    try {
        ContainerId cID = rmContainerEvent.getContainerId();
        ApplicationAttemptId applicationAttemptId = cID.getApplicationAttemptId();
        String applicationId = applicationAttemptId.getApplicationId().toString();
        String attemptId = applicationAttemptId.toString();

        Header header = Header.newBuilder()
            .withId(applicationId)
            .withApplicationID(applicationId)
            .withAttemptID(attemptId)
            .withContainerID(cID.toString())
            .build();

        RMContainerImpl rmc = (RMContainerImpl) rmContainerImpl;

        ResourceManagerEventProtos.ContainerEvent.Builder eventBuilder = ResourceManagerEventProtos.ContainerEvent.newBuilder()
            .setType(rmContainerEvent.getType().name())
            .setState(rmc.getState().name())
            .setStartTime(rmc.getCreationTime())
            .setLogUrl(rmc.getLogURL());

        if (rmc.getContainer() != null && rmc.getContainer().getResource() != null) {
            eventBuilder.setVcoresReserved(rmc.getContainer().getResource().getVirtualCores());
            eventBuilder.setMemoryReserved(rmc.getContainer().getResource().getMemory());
        }

        if (rmc.getAllocatedNode() != null) {
            eventBuilder.setContainerHostname(rmc.getAllocatedNode().getHost());
        }

        ContainerStatus containerStatus = (ContainerStatus) getField().get(rmContainerImpl);
        if (containerStatus != null) {
            eventBuilder
                .setIsFinished(true)
                .setExitStatus(rmc.getContainerExitStatus())
                .setReason(rmc.getDiagnosticsInfo())
                .setFinishTime(rmc.getFinishTime());
        }

        eventHandler.accept(System.currentTimeMillis(), header, eventBuilder.build());
    } catch (Throwable ignored) {
    }
}
 

/**
 * Create an event to record the start of an MR AppMaster
 *
 * @param appAttemptId
 *          the application attempt id.
 * @param startTime
 *          the start time of the AM.
 * @param containerId
 *          the containerId of the AM.
 * @param nodeManagerHost
 *          the node on which the AM is running.
 * @param nodeManagerPort
 *          the port on which the AM is running.
 * @param nodeManagerHttpPort
 *          the httpPort for the node running the AM.
 * @param forcedJobStateOnShutDown
 *          the state to force the job into
 */
public AMStartedEvent(ApplicationAttemptId appAttemptId, long startTime,
    ContainerId containerId, String nodeManagerHost, int nodeManagerPort,
    int nodeManagerHttpPort, String forcedJobStateOnShutDown,
    long submitTime) {
  datum.applicationAttemptId = new Utf8(appAttemptId.toString());
  datum.startTime = startTime;
  datum.containerId = new Utf8(containerId.toString());
  datum.nodeManagerHost = new Utf8(nodeManagerHost);
  datum.nodeManagerPort = nodeManagerPort;
  datum.nodeManagerHttpPort = nodeManagerHttpPort;
  this.forcedJobStateOnShutDown = forcedJobStateOnShutDown;
  this.submitTime = submitTime;
}
 

/**
 * Create an event to record the start of an MR AppMaster
 *
 * @param appAttemptId
 *          the application attempt id.
 * @param startTime
 *          the start time of the AM.
 * @param containerId
 *          the containerId of the AM.
 * @param nodeManagerHost
 *          the node on which the AM is running.
 * @param nodeManagerPort
 *          the port on which the AM is running.
 * @param nodeManagerHttpPort
 *          the httpPort for the node running the AM.
 * @param forcedJobStateOnShutDown
 *          the state to force the job into
 */
public AMStartedEvent(ApplicationAttemptId appAttemptId, long startTime,
    ContainerId containerId, String nodeManagerHost, int nodeManagerPort,
    int nodeManagerHttpPort, String forcedJobStateOnShutDown,
    long submitTime) {
  datum.applicationAttemptId = new Utf8(appAttemptId.toString());
  datum.startTime = startTime;
  datum.containerId = new Utf8(containerId.toString());
  datum.nodeManagerHost = new Utf8(nodeManagerHost);
  datum.nodeManagerPort = nodeManagerPort;
  datum.nodeManagerHttpPort = nodeManagerHttpPort;
  this.forcedJobStateOnShutDown = forcedJobStateOnShutDown;
  this.submitTime = submitTime;
}