Java Code Examples for org.apache.hadoop.yarn.util.ConverterUtils#toApplicationId()
The following examples show how to use
org.apache.hadoop.yarn.util.ConverterUtils#toApplicationId() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: RMWebServices.java From big-c with Apache License 2.0 | 6 votes |
private RMApp getRMAppForAppId(String appId) { if (appId == null || appId.isEmpty()) { throw new NotFoundException("appId, " + appId + ", is empty or null"); } ApplicationId id; try { id = ConverterUtils.toApplicationId(recordFactory, appId); } catch (NumberFormatException e) { throw new NotFoundException("appId is invalid"); } if (id == null) { throw new NotFoundException("appId is invalid"); } RMApp app = rm.getRMContext().getRMApps().get(id); if (app == null) { throw new NotFoundException("app with id: " + appId + " not found"); } return app; }
Example 2
Source File: ZKRMStateStore.java From hadoop with Apache License 2.0 | 6 votes |
private synchronized void loadRMAppState(RMState rmState) throws Exception { List<String> childNodes = getChildrenWithRetries(rmAppRoot, false); for (String childNodeName : childNodes) { String childNodePath = getNodePath(rmAppRoot, childNodeName); byte[] childData = getDataWithRetries(childNodePath, false); if (childNodeName.startsWith(ApplicationId.appIdStrPrefix)) { // application if (LOG.isDebugEnabled()) { LOG.debug("Loading application from znode: " + childNodeName); } ApplicationId appId = ConverterUtils.toApplicationId(childNodeName); ApplicationStateDataPBImpl appState = new ApplicationStateDataPBImpl( ApplicationStateDataProto.parseFrom(childData)); if (!appId.equals( appState.getApplicationSubmissionContext().getApplicationId())) { throw new YarnRuntimeException("The child node name is different " + "from the application id"); } rmState.appState.put(appId, appState); loadApplicationAttemptState(appState, appId); } else { LOG.info("Unknown child node with name: " + childNodeName); } } }
Example 3
Source File: NMWebServices.java From big-c with Apache License 2.0 | 6 votes |
@GET @Path("/apps/{appid}") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public AppInfo getNodeApp(@PathParam("appid") String appId) { init(); ApplicationId id = ConverterUtils.toApplicationId(recordFactory, appId); if (id == null) { throw new NotFoundException("app with id " + appId + " not found"); } Application app = this.nmContext.getApplications().get(id); if (app == null) { throw new NotFoundException("app with id " + appId + " not found"); } return new AppInfo(app); }
Example 4
Source File: ApplicationCLI.java From hadoop with Apache License 2.0 | 6 votes |
/** * Kills the application with the application id as appId * * @param applicationId * @throws YarnException * @throws IOException */ private void killApplication(String applicationId) throws YarnException, IOException { ApplicationId appId = ConverterUtils.toApplicationId(applicationId); ApplicationReport appReport = null; try { appReport = client.getApplicationReport(appId); } catch (ApplicationNotFoundException e) { sysout.println("Application with id '" + applicationId + "' doesn't exist in RM."); throw e; } if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED || appReport.getYarnApplicationState() == YarnApplicationState.KILLED || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) { sysout.println("Application " + applicationId + " has already finished "); } else { sysout.println("Killing application " + applicationId); client.killApplication(appId); } }
Example 5
Source File: RMWebServices.java From hadoop with Apache License 2.0 | 6 votes |
private RMApp getRMAppForAppId(String appId) { if (appId == null || appId.isEmpty()) { throw new NotFoundException("appId, " + appId + ", is empty or null"); } ApplicationId id; try { id = ConverterUtils.toApplicationId(recordFactory, appId); } catch (NumberFormatException e) { throw new NotFoundException("appId is invalid"); } if (id == null) { throw new NotFoundException("appId is invalid"); } RMApp app = rm.getRMContext().getRMApps().get(id); if (app == null) { throw new NotFoundException("app with id: " + appId + " not found"); } return app; }
Example 6
Source File: ApplicationPage.java From hadoop with Apache License 2.0 | 6 votes |
@Override protected void render(Block html) { ApplicationId applicationID = ConverterUtils.toApplicationId(this.recordFactory, $(APPLICATION_ID)); Application app = this.nmContext.getApplications().get(applicationID); AppInfo info = new AppInfo(app); info("Application's information") ._("ApplicationId", info.getId()) ._("ApplicationState", info.getState()) ._("User", info.getUser()); TABLE<Hamlet> containersListBody = html._(InfoBlock.class) .table("#containers"); for (String containerIdStr : info.getContainers()) { containersListBody .tr().td() .a(url("container", containerIdStr), containerIdStr) ._()._(); } containersListBody._(); }
Example 7
Source File: ApplicationCLI.java From big-c with Apache License 2.0 | 6 votes |
/** * Kills the application with the application id as appId * * @param applicationId * @throws YarnException * @throws IOException */ private void killApplication(String applicationId) throws YarnException, IOException { ApplicationId appId = ConverterUtils.toApplicationId(applicationId); ApplicationReport appReport = null; try { appReport = client.getApplicationReport(appId); } catch (ApplicationNotFoundException e) { sysout.println("Application with id '" + applicationId + "' doesn't exist in RM."); throw e; } if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED || appReport.getYarnApplicationState() == YarnApplicationState.KILLED || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) { sysout.println("Application " + applicationId + " has already finished "); } else { sysout.println("Killing application " + applicationId); client.killApplication(appId); } }
Example 8
Source File: FileSystemApplicationHistoryStore.java From big-c with Apache License 2.0 | 6 votes |
@Override public Map<ApplicationId, ApplicationHistoryData> getAllApplications() throws IOException { Map<ApplicationId, ApplicationHistoryData> historyDataMap = new HashMap<ApplicationId, ApplicationHistoryData>(); FileStatus[] files = fs.listStatus(rootDirPath); for (FileStatus file : files) { ApplicationId appId = ConverterUtils.toApplicationId(file.getPath().getName()); try { ApplicationHistoryData historyData = getApplication(appId); if (historyData != null) { historyDataMap.put(appId, historyData); } } catch (IOException e) { // Eat the exception not to disturb the getting the next // ApplicationHistoryData LOG.error("History information of application " + appId + " is not included into the result due to the exception", e); } } return historyDataMap; }
Example 9
Source File: LeveldbRMStateStore.java From big-c with Apache License 2.0 | 5 votes |
private ApplicationStateData createApplicationState(String appIdStr, byte[] data) throws IOException { ApplicationId appId = ConverterUtils.toApplicationId(appIdStr); ApplicationStateDataPBImpl appState = new ApplicationStateDataPBImpl( ApplicationStateDataProto.parseFrom(data)); if (!appId.equals( appState.getApplicationSubmissionContext().getApplicationId())) { throw new YarnRuntimeException("The database entry for " + appId + " contains data for " + appState.getApplicationSubmissionContext().getApplicationId()); } return appState; }
Example 10
Source File: NMLeveldbStateStoreService.java From big-c with Apache License 2.0 | 5 votes |
private RecoveredUserResources loadUserLocalizedResources( LeveldbIterator iter, String keyPrefix) throws IOException { RecoveredUserResources userResources = new RecoveredUserResources(); while (iter.hasNext()) { Entry<byte[],byte[]> entry = iter.peekNext(); String key = asString(entry.getKey()); if (!key.startsWith(keyPrefix)) { break; } if (key.startsWith(LOCALIZATION_FILECACHE_SUFFIX, keyPrefix.length())) { userResources.privateTrackerState = loadResourceTrackerState(iter, keyPrefix + LOCALIZATION_FILECACHE_SUFFIX); } else if (key.startsWith(LOCALIZATION_APPCACHE_SUFFIX, keyPrefix.length())) { int appIdStartPos = keyPrefix.length() + LOCALIZATION_APPCACHE_SUFFIX.length(); int appIdEndPos = key.indexOf('/', appIdStartPos); if (appIdEndPos < 0) { throw new IOException("Unable to determine appID in resource key: " + key); } ApplicationId appId = ConverterUtils.toApplicationId( key.substring(appIdStartPos, appIdEndPos)); userResources.appTrackerStates.put(appId, loadResourceTrackerState(iter, key.substring(0, appIdEndPos+1))); } else { throw new IOException("Unexpected user resource key " + key); } } return userResources; }
Example 11
Source File: RMWebServices.java From hadoop with Apache License 2.0 | 5 votes |
/** * Create the actual ApplicationSubmissionContext to be submitted to the RM * from the information provided by the user. * * @param newApp * the information provided by the user * @return returns the constructed ApplicationSubmissionContext * @throws IOException */ protected ApplicationSubmissionContext createAppSubmissionContext( ApplicationSubmissionContextInfo newApp) throws IOException { // create local resources and app submission context ApplicationId appid; String error = "Could not parse application id " + newApp.getApplicationId(); try { appid = ConverterUtils.toApplicationId(recordFactory, newApp.getApplicationId()); } catch (Exception e) { throw new BadRequestException(error); } ApplicationSubmissionContext appContext = ApplicationSubmissionContext.newInstance(appid, newApp.getApplicationName(), newApp.getQueue(), Priority.newInstance(newApp.getPriority()), createContainerLaunchContext(newApp), newApp.getUnmanagedAM(), newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(), createAppSubmissionContextResource(newApp), newApp.getApplicationType(), newApp.getKeepContainersAcrossApplicationAttempts(), newApp.getAppNodeLabelExpression(), newApp.getAMContainerNodeLabelExpression()); appContext.setApplicationTags(newApp.getApplicationTags()); return appContext; }
Example 12
Source File: RMWebServices.java From hadoop with Apache License 2.0 | 5 votes |
@GET @Path("/apps/{appid}/appattempts") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public AppAttemptsInfo getAppAttempts(@PathParam("appid") String appId) { init(); if (appId == null || appId.isEmpty()) { throw new NotFoundException("appId, " + appId + ", is empty or null"); } ApplicationId id; id = ConverterUtils.toApplicationId(recordFactory, appId); if (id == null) { throw new NotFoundException("appId is null"); } RMApp app = rm.getRMContext().getRMApps().get(id); if (app == null) { throw new NotFoundException("app with id: " + appId + " not found"); } AppAttemptsInfo appAttemptsInfo = new AppAttemptsInfo(); for (RMAppAttempt attempt : app.getAppAttempts().values()) { AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt, app.getUser()); appAttemptsInfo.add(attemptInfo); } return appAttemptsInfo; }
Example 13
Source File: ApplicationCLI.java From hadoop with Apache License 2.0 | 5 votes |
/** * Moves the application with the given ID to the given queue. */ private void moveApplicationAcrossQueues(String applicationId, String queue) throws YarnException, IOException { ApplicationId appId = ConverterUtils.toApplicationId(applicationId); ApplicationReport appReport = client.getApplicationReport(appId); if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED || appReport.getYarnApplicationState() == YarnApplicationState.KILLED || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) { sysout.println("Application " + applicationId + " has already finished "); } else { sysout.println("Moving application " + applicationId + " to queue " + queue); client.moveApplicationAcrossQueues(appId, queue); sysout.println("Successfully completed move."); } }
Example 14
Source File: WebServices.java From hadoop with Apache License 2.0 | 5 votes |
protected static ApplicationId parseApplicationId(String appId) { if (appId == null || appId.isEmpty()) { throw new NotFoundException("appId, " + appId + ", is empty or null"); } ApplicationId aid = ConverterUtils.toApplicationId(appId); if (aid == null) { throw new NotFoundException("appId is null"); } return aid; }
Example 15
Source File: RMWebServices.java From big-c with Apache License 2.0 | 5 votes |
@GET @Path("/apps/{appid}/appattempts") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public AppAttemptsInfo getAppAttempts(@PathParam("appid") String appId) { init(); if (appId == null || appId.isEmpty()) { throw new NotFoundException("appId, " + appId + ", is empty or null"); } ApplicationId id; id = ConverterUtils.toApplicationId(recordFactory, appId); if (id == null) { throw new NotFoundException("appId is null"); } RMApp app = rm.getRMContext().getRMApps().get(id); if (app == null) { throw new NotFoundException("app with id: " + appId + " not found"); } AppAttemptsInfo appAttemptsInfo = new AppAttemptsInfo(); for (RMAppAttempt attempt : app.getAppAttempts().values()) { AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt, app.getUser()); appAttemptsInfo.add(attemptInfo); } return appAttemptsInfo; }
Example 16
Source File: YarnClientProxy.java From PoseidonX with Apache License 2.0 | 5 votes |
/** * 根据APPID获得ApplicationReport * @param applicationId * @throws Exception */ public static ApplicationReport getApplicationReportByAppId(String applicationId) { if(StringUtils.isBlank(applicationId)){ return null; } ApplicationId appId = ConverterUtils.toApplicationId(applicationId); ApplicationReport report = null; try { report = yarnClient.getApplicationReport(appId); } catch (Exception e) { //LOGGER.error("YarnClientProxy getApplicationReportByAppId is error",e); } return report; }
Example 17
Source File: FlinkYarnSessionCli.java From flink with Apache License 2.0 | 5 votes |
@Override @Nullable public ApplicationId getClusterId(CommandLine commandLine) { if (commandLine.hasOption(applicationId.getOpt())) { return ConverterUtils.toApplicationId(commandLine.getOptionValue(applicationId.getOpt())); } else if (isYarnPropertiesFileMode(commandLine)) { return yarnApplicationIdFromYarnProperties; } else { return null; } }
Example 18
Source File: WebServices.java From big-c with Apache License 2.0 | 5 votes |
protected static ApplicationId parseApplicationId(String appId) { if (appId == null || appId.isEmpty()) { throw new NotFoundException("appId, " + appId + ", is empty or null"); } ApplicationId aid = ConverterUtils.toApplicationId(appId); if (aid == null) { throw new NotFoundException("appId is null"); } return aid; }
Example 19
Source File: FlinkYarnSessionCli.java From flink with Apache License 2.0 | 4 votes |
public FlinkYarnSessionCli( Configuration configuration, ClusterClientServiceLoader clusterClientServiceLoader, String configurationDirectory, String shortPrefix, String longPrefix, boolean acceptInteractiveInput) throws FlinkException { super(configuration); this.clusterClientServiceLoader = checkNotNull(clusterClientServiceLoader); this.configurationDirectory = checkNotNull(configurationDirectory); this.acceptInteractiveInput = acceptInteractiveInput; // Create the command line options query = new Option(shortPrefix + "q", longPrefix + "query", false, "Display available YARN resources (memory, cores)"); applicationId = new Option(shortPrefix + "id", longPrefix + "applicationId", true, "Attach to running YARN session"); queue = new Option(shortPrefix + "qu", longPrefix + "queue", true, "Specify YARN queue."); shipPath = new Option(shortPrefix + "t", longPrefix + "ship", true, "Ship files in the specified directory (t for transfer)"); flinkJar = new Option(shortPrefix + "j", longPrefix + "jar", true, "Path to Flink jar file"); jmMemory = new Option(shortPrefix + "jm", longPrefix + "jobManagerMemory", true, "Memory for JobManager Container with optional unit (default: MB)"); tmMemory = new Option(shortPrefix + "tm", longPrefix + "taskManagerMemory", true, "Memory per TaskManager Container with optional unit (default: MB)"); slots = new Option(shortPrefix + "s", longPrefix + "slots", true, "Number of slots per TaskManager"); dynamicproperties = Option.builder(shortPrefix + "D") .argName("property=value") .numberOfArgs(2) .valueSeparator() .desc("use value for given property") .build(); name = new Option(shortPrefix + "nm", longPrefix + "name", true, "Set a custom name for the application on YARN"); applicationType = new Option(shortPrefix + "at", longPrefix + "applicationType", true, "Set a custom application type for the application on YARN"); zookeeperNamespace = new Option(shortPrefix + "z", longPrefix + "zookeeperNamespace", true, "Namespace to create the Zookeeper sub-paths for high availability mode"); nodeLabel = new Option(shortPrefix + "nl", longPrefix + "nodeLabel", true, "Specify YARN node label for the YARN application"); help = new Option(shortPrefix + "h", longPrefix + "help", false, "Help for the Yarn session CLI."); allOptions = new Options(); allOptions.addOption(flinkJar); allOptions.addOption(jmMemory); allOptions.addOption(tmMemory); allOptions.addOption(queue); allOptions.addOption(query); allOptions.addOption(shipPath); allOptions.addOption(slots); allOptions.addOption(dynamicproperties); allOptions.addOption(DETACHED_OPTION); allOptions.addOption(YARN_DETACHED_OPTION); allOptions.addOption(name); allOptions.addOption(applicationId); allOptions.addOption(applicationType); allOptions.addOption(zookeeperNamespace); allOptions.addOption(nodeLabel); allOptions.addOption(help); // try loading a potential yarn properties file this.yarnPropertiesFileLocation = configuration.getString(YarnConfigOptions.PROPERTIES_FILE_LOCATION); final File yarnPropertiesLocation = getYarnPropertiesLocation(yarnPropertiesFileLocation); yarnPropertiesFile = new Properties(); if (yarnPropertiesLocation.exists()) { LOG.info("Found Yarn properties file under {}.", yarnPropertiesLocation.getAbsolutePath()); try (InputStream is = new FileInputStream(yarnPropertiesLocation)) { yarnPropertiesFile.load(is); } catch (IOException ioe) { throw new FlinkException("Could not read the Yarn properties file " + yarnPropertiesLocation + ". Please delete the file at " + yarnPropertiesLocation.getAbsolutePath() + '.', ioe); } final String yarnApplicationIdString = yarnPropertiesFile.getProperty(YARN_APPLICATION_ID_KEY); if (yarnApplicationIdString == null) { throw new FlinkException("Yarn properties file found but doesn't contain a " + "Yarn application id. Please delete the file at " + yarnPropertiesLocation.getAbsolutePath()); } try { // try converting id to ApplicationId yarnApplicationIdFromYarnProperties = ConverterUtils.toApplicationId(yarnApplicationIdString); } catch (Exception e) { throw new FlinkException("YARN properties contain an invalid entry for " + "application id: " + yarnApplicationIdString + ". Please delete the file at " + yarnPropertiesLocation.getAbsolutePath(), e); } } else { yarnApplicationIdFromYarnProperties = null; } }
Example 20
Source File: FlinkYarnSessionCli.java From flink with Apache License 2.0 | 4 votes |
public FlinkYarnSessionCli( Configuration configuration, String configurationDirectory, String shortPrefix, String longPrefix, boolean acceptInteractiveInput) throws FlinkException { super(configuration); this.configurationDirectory = Preconditions.checkNotNull(configurationDirectory); this.acceptInteractiveInput = acceptInteractiveInput; // Create the command line options query = new Option(shortPrefix + "q", longPrefix + "query", false, "Display available YARN resources (memory, cores)"); applicationId = new Option(shortPrefix + "id", longPrefix + "applicationId", true, "Attach to running YARN session"); queue = new Option(shortPrefix + "qu", longPrefix + "queue", true, "Specify YARN queue."); shipPath = new Option(shortPrefix + "t", longPrefix + "ship", true, "Ship files in the specified directory (t for transfer)"); flinkJar = new Option(shortPrefix + "j", longPrefix + "jar", true, "Path to Flink jar file"); jmMemory = new Option(shortPrefix + "jm", longPrefix + "jobManagerMemory", true, "Memory for JobManager Container with optional unit (default: MB)"); tmMemory = new Option(shortPrefix + "tm", longPrefix + "taskManagerMemory", true, "Memory per TaskManager Container with optional unit (default: MB)"); container = new Option(shortPrefix + "n", longPrefix + "container", true, "Number of YARN container to allocate (=Number of Task Managers)"); slots = new Option(shortPrefix + "s", longPrefix + "slots", true, "Number of slots per TaskManager"); dynamicproperties = Option.builder(shortPrefix + "D") .argName("property=value") .numberOfArgs(2) .valueSeparator() .desc("use value for given property") .build(); streaming = new Option(shortPrefix + "st", longPrefix + "streaming", false, "Start Flink in streaming mode"); name = new Option(shortPrefix + "nm", longPrefix + "name", true, "Set a custom name for the application on YARN"); applicationType = new Option(shortPrefix + "at", longPrefix + "applicationType", true, "Set a custom application type for the application on YARN"); zookeeperNamespace = new Option(shortPrefix + "z", longPrefix + "zookeeperNamespace", true, "Namespace to create the Zookeeper sub-paths for high availability mode"); nodeLabel = new Option(shortPrefix + "nl", longPrefix + "nodeLabel", true, "Specify YARN node label for the YARN application"); help = new Option(shortPrefix + "h", longPrefix + "help", false, "Help for the Yarn session CLI."); allOptions = new Options(); allOptions.addOption(flinkJar); allOptions.addOption(jmMemory); allOptions.addOption(tmMemory); allOptions.addOption(container); allOptions.addOption(queue); allOptions.addOption(query); allOptions.addOption(shipPath); allOptions.addOption(slots); allOptions.addOption(dynamicproperties); allOptions.addOption(DETACHED_OPTION); allOptions.addOption(SHUTDOWN_IF_ATTACHED_OPTION); allOptions.addOption(YARN_DETACHED_OPTION); allOptions.addOption(streaming); allOptions.addOption(name); allOptions.addOption(applicationId); allOptions.addOption(applicationType); allOptions.addOption(zookeeperNamespace); allOptions.addOption(nodeLabel); allOptions.addOption(help); // try loading a potential yarn properties file this.yarnPropertiesFileLocation = configuration.getString(YarnConfigOptions.PROPERTIES_FILE_LOCATION); final File yarnPropertiesLocation = getYarnPropertiesLocation(yarnPropertiesFileLocation); yarnPropertiesFile = new Properties(); if (yarnPropertiesLocation.exists()) { LOG.info("Found Yarn properties file under {}.", yarnPropertiesLocation.getAbsolutePath()); try (InputStream is = new FileInputStream(yarnPropertiesLocation)) { yarnPropertiesFile.load(is); } catch (IOException ioe) { throw new FlinkException("Could not read the Yarn properties file " + yarnPropertiesLocation + ". Please delete the file at " + yarnPropertiesLocation.getAbsolutePath() + '.', ioe); } final String yarnApplicationIdString = yarnPropertiesFile.getProperty(YARN_APPLICATION_ID_KEY); if (yarnApplicationIdString == null) { throw new FlinkException("Yarn properties file found but doesn't contain a " + "Yarn application id. Please delete the file at " + yarnPropertiesLocation.getAbsolutePath()); } try { // try converting id to ApplicationId yarnApplicationIdFromYarnProperties = ConverterUtils.toApplicationId(yarnApplicationIdString); } catch (Exception e) { throw new FlinkException("YARN properties contain an invalid entry for " + "application id: " + yarnApplicationIdString + ". Please delete the file at " + yarnPropertiesLocation.getAbsolutePath(), e); } } else { yarnApplicationIdFromYarnProperties = null; } this.yarnConfiguration = new YarnConfiguration(); }