Java Code Examples for org.apache.hadoop.yarn.util.ConverterUtils#toApplicationId()

private RMApp getRMAppForAppId(String appId) {

    if (appId == null || appId.isEmpty()) {
      throw new NotFoundException("appId, " + appId + ", is empty or null");
    }
    ApplicationId id;
    try {
      id = ConverterUtils.toApplicationId(recordFactory, appId);
    } catch (NumberFormatException e) {
      throw new NotFoundException("appId is invalid");
    }
    if (id == null) {
      throw new NotFoundException("appId is invalid");
    }
    RMApp app = rm.getRMContext().getRMApps().get(id);
    if (app == null) {
      throw new NotFoundException("app with id: " + appId + " not found");
    }
    return app;
  }
 

private synchronized void loadRMAppState(RMState rmState) throws Exception {
  List<String> childNodes = getChildrenWithRetries(rmAppRoot, false);
  for (String childNodeName : childNodes) {
    String childNodePath = getNodePath(rmAppRoot, childNodeName);
    byte[] childData = getDataWithRetries(childNodePath, false);
    if (childNodeName.startsWith(ApplicationId.appIdStrPrefix)) {
      // application
      if (LOG.isDebugEnabled()) {
        LOG.debug("Loading application from znode: " + childNodeName);
      }
      ApplicationId appId = ConverterUtils.toApplicationId(childNodeName);
      ApplicationStateDataPBImpl appState =
          new ApplicationStateDataPBImpl(
              ApplicationStateDataProto.parseFrom(childData));
      if (!appId.equals(
          appState.getApplicationSubmissionContext().getApplicationId())) {
        throw new YarnRuntimeException("The child node name is different " +
            "from the application id");
      }
      rmState.appState.put(appId, appState);
      loadApplicationAttemptState(appState, appId);
    } else {
      LOG.info("Unknown child node with name: " + childNodeName);
    }
  }
}
 

@GET
@Path("/apps/{appid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public AppInfo getNodeApp(@PathParam("appid") String appId) {
  init();
  ApplicationId id = ConverterUtils.toApplicationId(recordFactory, appId);
  if (id == null) {
    throw new NotFoundException("app with id " + appId + " not found");
  }
  Application app = this.nmContext.getApplications().get(id);
  if (app == null) {
    throw new NotFoundException("app with id " + appId + " not found");
  }
  return new AppInfo(app);

}
 

/**
 * Kills the application with the application id as appId
 * 
 * @param applicationId
 * @throws YarnException
 * @throws IOException
 */
private void killApplication(String applicationId) throws YarnException,
    IOException {
  ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
  ApplicationReport  appReport = null;
  try {
    appReport = client.getApplicationReport(appId);
  } catch (ApplicationNotFoundException e) {
    sysout.println("Application with id '" + applicationId +
        "' doesn't exist in RM.");
    throw e;
  }

  if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
      || appReport.getYarnApplicationState() == YarnApplicationState.KILLED
      || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
    sysout.println("Application " + applicationId + " has already finished ");
  } else {
    sysout.println("Killing application " + applicationId);
    client.killApplication(appId);
  }
}
 

private RMApp getRMAppForAppId(String appId) {

    if (appId == null || appId.isEmpty()) {
      throw new NotFoundException("appId, " + appId + ", is empty or null");
    }
    ApplicationId id;
    try {
      id = ConverterUtils.toApplicationId(recordFactory, appId);
    } catch (NumberFormatException e) {
      throw new NotFoundException("appId is invalid");
    }
    if (id == null) {
      throw new NotFoundException("appId is invalid");
    }
    RMApp app = rm.getRMContext().getRMApps().get(id);
    if (app == null) {
      throw new NotFoundException("app with id: " + appId + " not found");
    }
    return app;
  }
 

@Override
protected void render(Block html) {
  ApplicationId applicationID =
      ConverterUtils.toApplicationId(this.recordFactory,
          $(APPLICATION_ID));
  Application app = this.nmContext.getApplications().get(applicationID);
  AppInfo info = new AppInfo(app);
  info("Application's information")
        ._("ApplicationId", info.getId())
        ._("ApplicationState", info.getState())
        ._("User", info.getUser());
  TABLE<Hamlet> containersListBody = html._(InfoBlock.class)
      .table("#containers");
  for (String containerIdStr : info.getContainers()) {
    containersListBody
           .tr().td()
             .a(url("container", containerIdStr), containerIdStr)
             ._()._();
  }
  containersListBody._();
}
 

/**
 * Kills the application with the application id as appId
 * 
 * @param applicationId
 * @throws YarnException
 * @throws IOException
 */
private void killApplication(String applicationId) throws YarnException,
    IOException {
  ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
  ApplicationReport  appReport = null;
  try {
    appReport = client.getApplicationReport(appId);
  } catch (ApplicationNotFoundException e) {
    sysout.println("Application with id '" + applicationId +
        "' doesn't exist in RM.");
    throw e;
  }

  if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
      || appReport.getYarnApplicationState() == YarnApplicationState.KILLED
      || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
    sysout.println("Application " + applicationId + " has already finished ");
  } else {
    sysout.println("Killing application " + applicationId);
    client.killApplication(appId);
  }
}
 

@Override
public Map<ApplicationId, ApplicationHistoryData> getAllApplications()
    throws IOException {
  Map<ApplicationId, ApplicationHistoryData> historyDataMap =
      new HashMap<ApplicationId, ApplicationHistoryData>();
  FileStatus[] files = fs.listStatus(rootDirPath);
  for (FileStatus file : files) {
    ApplicationId appId =
        ConverterUtils.toApplicationId(file.getPath().getName());
    try {
      ApplicationHistoryData historyData = getApplication(appId);
      if (historyData != null) {
        historyDataMap.put(appId, historyData);
      }
    } catch (IOException e) {
      // Eat the exception not to disturb the getting the next
      // ApplicationHistoryData
      LOG.error("History information of application " + appId
          + " is not included into the result due to the exception", e);
    }
  }
  return historyDataMap;
}
 

private ApplicationStateData createApplicationState(String appIdStr,
    byte[] data) throws IOException {
  ApplicationId appId = ConverterUtils.toApplicationId(appIdStr);
  ApplicationStateDataPBImpl appState =
      new ApplicationStateDataPBImpl(
          ApplicationStateDataProto.parseFrom(data));
  if (!appId.equals(
      appState.getApplicationSubmissionContext().getApplicationId())) {
    throw new YarnRuntimeException("The database entry for " + appId
        + " contains data for "
        + appState.getApplicationSubmissionContext().getApplicationId());
  }
  return appState;
}
 

private RecoveredUserResources loadUserLocalizedResources(
    LeveldbIterator iter, String keyPrefix) throws IOException {
  RecoveredUserResources userResources = new RecoveredUserResources();
  while (iter.hasNext()) {
    Entry<byte[],byte[]> entry = iter.peekNext();
    String key = asString(entry.getKey());
    if (!key.startsWith(keyPrefix)) {
      break;
    }

    if (key.startsWith(LOCALIZATION_FILECACHE_SUFFIX, keyPrefix.length())) {
      userResources.privateTrackerState = loadResourceTrackerState(iter,
          keyPrefix + LOCALIZATION_FILECACHE_SUFFIX);
    } else if (key.startsWith(LOCALIZATION_APPCACHE_SUFFIX,
        keyPrefix.length())) {
      int appIdStartPos = keyPrefix.length() +
          LOCALIZATION_APPCACHE_SUFFIX.length();
      int appIdEndPos = key.indexOf('/', appIdStartPos);
      if (appIdEndPos < 0) {
        throw new IOException("Unable to determine appID in resource key: "
            + key);
      }
      ApplicationId appId = ConverterUtils.toApplicationId(
          key.substring(appIdStartPos, appIdEndPos));
      userResources.appTrackerStates.put(appId,
          loadResourceTrackerState(iter, key.substring(0, appIdEndPos+1)));
    } else {
      throw new IOException("Unexpected user resource key " + key);
    }
  }
  return userResources;
}
 

/**
 * Create the actual ApplicationSubmissionContext to be submitted to the RM
 * from the information provided by the user.
 * 
 * @param newApp
 *          the information provided by the user
 * @return returns the constructed ApplicationSubmissionContext
 * @throws IOException
 */
protected ApplicationSubmissionContext createAppSubmissionContext(
    ApplicationSubmissionContextInfo newApp) throws IOException {

  // create local resources and app submission context

  ApplicationId appid;
  String error =
      "Could not parse application id " + newApp.getApplicationId();
  try {
    appid =
        ConverterUtils.toApplicationId(recordFactory,
          newApp.getApplicationId());
  } catch (Exception e) {
    throw new BadRequestException(error);
  }
  ApplicationSubmissionContext appContext =
      ApplicationSubmissionContext.newInstance(appid,
        newApp.getApplicationName(), newApp.getQueue(),
        Priority.newInstance(newApp.getPriority()),
        createContainerLaunchContext(newApp), newApp.getUnmanagedAM(),
        newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(),
        createAppSubmissionContextResource(newApp),
        newApp.getApplicationType(),
        newApp.getKeepContainersAcrossApplicationAttempts(),
        newApp.getAppNodeLabelExpression(),
        newApp.getAMContainerNodeLabelExpression());
  appContext.setApplicationTags(newApp.getApplicationTags());

  return appContext;
}
 

@GET
@Path("/apps/{appid}/appattempts")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public AppAttemptsInfo getAppAttempts(@PathParam("appid") String appId) {

  init();
  if (appId == null || appId.isEmpty()) {
    throw new NotFoundException("appId, " + appId + ", is empty or null");
  }
  ApplicationId id;
  id = ConverterUtils.toApplicationId(recordFactory, appId);
  if (id == null) {
    throw new NotFoundException("appId is null");
  }
  RMApp app = rm.getRMContext().getRMApps().get(id);
  if (app == null) {
    throw new NotFoundException("app with id: " + appId + " not found");
  }

  AppAttemptsInfo appAttemptsInfo = new AppAttemptsInfo();
  for (RMAppAttempt attempt : app.getAppAttempts().values()) {
    AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt, app.getUser());
    appAttemptsInfo.add(attemptInfo);
  }

  return appAttemptsInfo;
}
 

/**
 * Moves the application with the given ID to the given queue.
 */
private void moveApplicationAcrossQueues(String applicationId, String queue)
    throws YarnException, IOException {
  ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
  ApplicationReport appReport = client.getApplicationReport(appId);
  if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
      || appReport.getYarnApplicationState() == YarnApplicationState.KILLED
      || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
    sysout.println("Application " + applicationId + " has already finished ");
  } else {
    sysout.println("Moving application " + applicationId + " to queue " + queue);
    client.moveApplicationAcrossQueues(appId, queue);
    sysout.println("Successfully completed move.");
  }
}
 

protected static ApplicationId parseApplicationId(String appId) {
  if (appId == null || appId.isEmpty()) {
    throw new NotFoundException("appId, " + appId + ", is empty or null");
  }
  ApplicationId aid = ConverterUtils.toApplicationId(appId);
  if (aid == null) {
    throw new NotFoundException("appId is null");
  }
  return aid;
}
 

@GET
@Path("/apps/{appid}/appattempts")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public AppAttemptsInfo getAppAttempts(@PathParam("appid") String appId) {

  init();
  if (appId == null || appId.isEmpty()) {
    throw new NotFoundException("appId, " + appId + ", is empty or null");
  }
  ApplicationId id;
  id = ConverterUtils.toApplicationId(recordFactory, appId);
  if (id == null) {
    throw new NotFoundException("appId is null");
  }
  RMApp app = rm.getRMContext().getRMApps().get(id);
  if (app == null) {
    throw new NotFoundException("app with id: " + appId + " not found");
  }

  AppAttemptsInfo appAttemptsInfo = new AppAttemptsInfo();
  for (RMAppAttempt attempt : app.getAppAttempts().values()) {
    AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt, app.getUser());
    appAttemptsInfo.add(attemptInfo);
  }

  return appAttemptsInfo;
}
 

/**
 * 根据APPID获得ApplicationReport
 * @param applicationId
 * @throws Exception
 */
public static ApplicationReport getApplicationReportByAppId(String applicationId) {
    if(StringUtils.isBlank(applicationId)){
        return null;
    }
    ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
    ApplicationReport report = null;
    try {
        report = yarnClient.getApplicationReport(appId);
    } catch (Exception e) {
        //LOGGER.error("YarnClientProxy getApplicationReportByAppId is error",e);
    }
    return report;
}
 

@Override
@Nullable
public ApplicationId getClusterId(CommandLine commandLine) {
	if (commandLine.hasOption(applicationId.getOpt())) {
		return ConverterUtils.toApplicationId(commandLine.getOptionValue(applicationId.getOpt()));
	} else if (isYarnPropertiesFileMode(commandLine)) {
		return yarnApplicationIdFromYarnProperties;
	} else {
		return null;
	}
}
 

protected static ApplicationId parseApplicationId(String appId) {
  if (appId == null || appId.isEmpty()) {
    throw new NotFoundException("appId, " + appId + ", is empty or null");
  }
  ApplicationId aid = ConverterUtils.toApplicationId(appId);
  if (aid == null) {
    throw new NotFoundException("appId is null");
  }
  return aid;
}
 

public FlinkYarnSessionCli(
		Configuration configuration,
		ClusterClientServiceLoader clusterClientServiceLoader,
		String configurationDirectory,
		String shortPrefix,
		String longPrefix,
		boolean acceptInteractiveInput) throws FlinkException {
	super(configuration);
	this.clusterClientServiceLoader = checkNotNull(clusterClientServiceLoader);
	this.configurationDirectory = checkNotNull(configurationDirectory);
	this.acceptInteractiveInput = acceptInteractiveInput;

	// Create the command line options

	query = new Option(shortPrefix + "q", longPrefix + "query", false, "Display available YARN resources (memory, cores)");
	applicationId = new Option(shortPrefix + "id", longPrefix + "applicationId", true, "Attach to running YARN session");
	queue = new Option(shortPrefix + "qu", longPrefix + "queue", true, "Specify YARN queue.");
	shipPath = new Option(shortPrefix + "t", longPrefix + "ship", true, "Ship files in the specified directory (t for transfer)");
	flinkJar = new Option(shortPrefix + "j", longPrefix + "jar", true, "Path to Flink jar file");
	jmMemory = new Option(shortPrefix + "jm", longPrefix + "jobManagerMemory", true, "Memory for JobManager Container with optional unit (default: MB)");
	tmMemory = new Option(shortPrefix + "tm", longPrefix + "taskManagerMemory", true, "Memory per TaskManager Container with optional unit (default: MB)");
	slots = new Option(shortPrefix + "s", longPrefix + "slots", true, "Number of slots per TaskManager");
	dynamicproperties = Option.builder(shortPrefix + "D")
		.argName("property=value")
		.numberOfArgs(2)
		.valueSeparator()
		.desc("use value for given property")
		.build();
	name = new Option(shortPrefix + "nm", longPrefix + "name", true, "Set a custom name for the application on YARN");
	applicationType = new Option(shortPrefix + "at", longPrefix + "applicationType", true, "Set a custom application type for the application on YARN");
	zookeeperNamespace = new Option(shortPrefix + "z", longPrefix + "zookeeperNamespace", true, "Namespace to create the Zookeeper sub-paths for high availability mode");
	nodeLabel = new Option(shortPrefix + "nl", longPrefix + "nodeLabel", true, "Specify YARN node label for the YARN application");
	help = new Option(shortPrefix + "h", longPrefix + "help", false, "Help for the Yarn session CLI.");

	allOptions = new Options();
	allOptions.addOption(flinkJar);
	allOptions.addOption(jmMemory);
	allOptions.addOption(tmMemory);
	allOptions.addOption(queue);
	allOptions.addOption(query);
	allOptions.addOption(shipPath);
	allOptions.addOption(slots);
	allOptions.addOption(dynamicproperties);
	allOptions.addOption(DETACHED_OPTION);
	allOptions.addOption(YARN_DETACHED_OPTION);
	allOptions.addOption(name);
	allOptions.addOption(applicationId);
	allOptions.addOption(applicationType);
	allOptions.addOption(zookeeperNamespace);
	allOptions.addOption(nodeLabel);
	allOptions.addOption(help);

	// try loading a potential yarn properties file
	this.yarnPropertiesFileLocation = configuration.getString(YarnConfigOptions.PROPERTIES_FILE_LOCATION);
	final File yarnPropertiesLocation = getYarnPropertiesLocation(yarnPropertiesFileLocation);

	yarnPropertiesFile = new Properties();

	if (yarnPropertiesLocation.exists()) {
		LOG.info("Found Yarn properties file under {}.", yarnPropertiesLocation.getAbsolutePath());

		try (InputStream is = new FileInputStream(yarnPropertiesLocation)) {
			yarnPropertiesFile.load(is);
		} catch (IOException ioe) {
			throw new FlinkException("Could not read the Yarn properties file " + yarnPropertiesLocation +
				". Please delete the file at " + yarnPropertiesLocation.getAbsolutePath() + '.', ioe);
		}

		final String yarnApplicationIdString = yarnPropertiesFile.getProperty(YARN_APPLICATION_ID_KEY);

		if (yarnApplicationIdString == null) {
			throw new FlinkException("Yarn properties file found but doesn't contain a " +
				"Yarn application id. Please delete the file at " + yarnPropertiesLocation.getAbsolutePath());
		}

		try {
			// try converting id to ApplicationId
			yarnApplicationIdFromYarnProperties = ConverterUtils.toApplicationId(yarnApplicationIdString);
		}
		catch (Exception e) {
			throw new FlinkException("YARN properties contain an invalid entry for " +
				"application id: " + yarnApplicationIdString + ". Please delete the file at " +
				yarnPropertiesLocation.getAbsolutePath(), e);
		}
	} else {
		yarnApplicationIdFromYarnProperties = null;
	}
}
 

public FlinkYarnSessionCli(
		Configuration configuration,
		String configurationDirectory,
		String shortPrefix,
		String longPrefix,
		boolean acceptInteractiveInput) throws FlinkException {
	super(configuration);
	this.configurationDirectory = Preconditions.checkNotNull(configurationDirectory);
	this.acceptInteractiveInput = acceptInteractiveInput;

	// Create the command line options

	query = new Option(shortPrefix + "q", longPrefix + "query", false, "Display available YARN resources (memory, cores)");
	applicationId = new Option(shortPrefix + "id", longPrefix + "applicationId", true, "Attach to running YARN session");
	queue = new Option(shortPrefix + "qu", longPrefix + "queue", true, "Specify YARN queue.");
	shipPath = new Option(shortPrefix + "t", longPrefix + "ship", true, "Ship files in the specified directory (t for transfer)");
	flinkJar = new Option(shortPrefix + "j", longPrefix + "jar", true, "Path to Flink jar file");
	jmMemory = new Option(shortPrefix + "jm", longPrefix + "jobManagerMemory", true, "Memory for JobManager Container with optional unit (default: MB)");
	tmMemory = new Option(shortPrefix + "tm", longPrefix + "taskManagerMemory", true, "Memory per TaskManager Container with optional unit (default: MB)");
	container = new Option(shortPrefix + "n", longPrefix + "container", true, "Number of YARN container to allocate (=Number of Task Managers)");
	slots = new Option(shortPrefix + "s", longPrefix + "slots", true, "Number of slots per TaskManager");
	dynamicproperties = Option.builder(shortPrefix + "D")
		.argName("property=value")
		.numberOfArgs(2)
		.valueSeparator()
		.desc("use value for given property")
		.build();
	streaming = new Option(shortPrefix + "st", longPrefix + "streaming", false, "Start Flink in streaming mode");
	name = new Option(shortPrefix + "nm", longPrefix + "name", true, "Set a custom name for the application on YARN");
	applicationType = new Option(shortPrefix + "at", longPrefix + "applicationType", true, "Set a custom application type for the application on YARN");
	zookeeperNamespace = new Option(shortPrefix + "z", longPrefix + "zookeeperNamespace", true, "Namespace to create the Zookeeper sub-paths for high availability mode");
	nodeLabel = new Option(shortPrefix + "nl", longPrefix + "nodeLabel", true, "Specify YARN node label for the YARN application");
	help = new Option(shortPrefix + "h", longPrefix + "help", false, "Help for the Yarn session CLI.");

	allOptions = new Options();
	allOptions.addOption(flinkJar);
	allOptions.addOption(jmMemory);
	allOptions.addOption(tmMemory);
	allOptions.addOption(container);
	allOptions.addOption(queue);
	allOptions.addOption(query);
	allOptions.addOption(shipPath);
	allOptions.addOption(slots);
	allOptions.addOption(dynamicproperties);
	allOptions.addOption(DETACHED_OPTION);
	allOptions.addOption(SHUTDOWN_IF_ATTACHED_OPTION);
	allOptions.addOption(YARN_DETACHED_OPTION);
	allOptions.addOption(streaming);
	allOptions.addOption(name);
	allOptions.addOption(applicationId);
	allOptions.addOption(applicationType);
	allOptions.addOption(zookeeperNamespace);
	allOptions.addOption(nodeLabel);
	allOptions.addOption(help);

	// try loading a potential yarn properties file
	this.yarnPropertiesFileLocation = configuration.getString(YarnConfigOptions.PROPERTIES_FILE_LOCATION);
	final File yarnPropertiesLocation = getYarnPropertiesLocation(yarnPropertiesFileLocation);

	yarnPropertiesFile = new Properties();

	if (yarnPropertiesLocation.exists()) {
		LOG.info("Found Yarn properties file under {}.", yarnPropertiesLocation.getAbsolutePath());

		try (InputStream is = new FileInputStream(yarnPropertiesLocation)) {
			yarnPropertiesFile.load(is);
		} catch (IOException ioe) {
			throw new FlinkException("Could not read the Yarn properties file " + yarnPropertiesLocation +
				". Please delete the file at " + yarnPropertiesLocation.getAbsolutePath() + '.', ioe);
		}

		final String yarnApplicationIdString = yarnPropertiesFile.getProperty(YARN_APPLICATION_ID_KEY);

		if (yarnApplicationIdString == null) {
			throw new FlinkException("Yarn properties file found but doesn't contain a " +
				"Yarn application id. Please delete the file at " + yarnPropertiesLocation.getAbsolutePath());
		}

		try {
			// try converting id to ApplicationId
			yarnApplicationIdFromYarnProperties = ConverterUtils.toApplicationId(yarnApplicationIdString);
		}
		catch (Exception e) {
			throw new FlinkException("YARN properties contain an invalid entry for " +
				"application id: " + yarnApplicationIdString + ". Please delete the file at " +
				yarnPropertiesLocation.getAbsolutePath(), e);
		}
	} else {
		yarnApplicationIdFromYarnProperties = null;
	}

	this.yarnConfiguration = new YarnConfiguration();
}