Java Code Examples for org.apache.hadoop.yarn.util.ConverterUtils

The following examples show how to use org.apache.hadoop.yarn.util.ConverterUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestMRAppMaster.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMRAppMasterForDifferentUser() throws IOException,
    InterruptedException {
  String applicationAttemptIdStr = "appattempt_1317529182569_0004_000001";
  String containerIdStr = "container_1317529182569_0004_000001_1";
  
  String userName = "TestAppMasterUser";
  ApplicationAttemptId applicationAttemptId = ConverterUtils
      .toApplicationAttemptId(applicationAttemptIdStr);
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  MRAppMasterTest appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
          System.currentTimeMillis());
  JobConf conf = new JobConf();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
  Path userPath = new Path(stagingDir, userName);
  Path userStagingPath = new Path(userPath, ".staging");
  assertEquals(userStagingPath.toString(),
    appMaster.stagingDirPath.toString());
}
 
Example 2
Source Project: big-c   Source File: RMWebServices.java    License: Apache License 2.0 6 votes vote down vote up
private RMApp getRMAppForAppId(String appId) {

    if (appId == null || appId.isEmpty()) {
      throw new NotFoundException("appId, " + appId + ", is empty or null");
    }
    ApplicationId id;
    try {
      id = ConverterUtils.toApplicationId(recordFactory, appId);
    } catch (NumberFormatException e) {
      throw new NotFoundException("appId is invalid");
    }
    if (id == null) {
      throw new NotFoundException("appId is invalid");
    }
    RMApp app = rm.getRMContext().getRMApps().get(id);
    if (app == null) {
      throw new NotFoundException("app with id: " + appId + " not found");
    }
    return app;
  }
 
Example 3
Source Project: big-c   Source File: ContainerManagementProtocolProxy.java    License: Apache License 2.0 6 votes vote down vote up
@Private
@VisibleForTesting
protected ContainerManagementProtocol newProxy(final YarnRPC rpc,
    String containerManagerBindAddr, ContainerId containerId, Token token)
    throws InvalidToken {

  if (token == null) {
    throw new InvalidToken("No NMToken sent for "
        + containerManagerBindAddr);
  }
  
  final InetSocketAddress cmAddr =
      NetUtils.createSocketAddr(containerManagerBindAddr);
  LOG.info("Opening proxy : " + containerManagerBindAddr);
  // the user in createRemoteUser in this context has to be ContainerID
  UserGroupInformation user =
      UserGroupInformation.createRemoteUser(containerId
          .getApplicationAttemptId().toString());

  org.apache.hadoop.security.token.Token<NMTokenIdentifier> nmToken =
      ConverterUtils.convertFromYarn(token, cmAddr);
  user.addToken(nmToken);

  return NMProxy.createNMProxy(conf, ContainerManagementProtocol.class,
    user, rpc, cmAddr);
}
 
Example 4
Source Project: hadoop   Source File: AggregatedLogsBlock.java    License: Apache License 2.0 6 votes vote down vote up
private NodeId verifyAndGetNodeId(Block html) {
  String nodeIdStr = $(NM_NODENAME);
  if (nodeIdStr == null || nodeIdStr.isEmpty()) {
    html.h1()._("Cannot get container logs without a NodeId")._();
    return null;
  }
  NodeId nodeId = null;
  try {
    nodeId = ConverterUtils.toNodeId(nodeIdStr);
  } catch (IllegalArgumentException e) {
    html.h1()._("Cannot get container logs. Invalid nodeId: " + nodeIdStr)
        ._();
    return null;
  }
  return nodeId;
}
 
Example 5
Source Project: big-c   Source File: ZKRMStateStore.java    License: Apache License 2.0 6 votes vote down vote up
private synchronized void loadRMAppState(RMState rmState) throws Exception {
  List<String> childNodes = getChildrenWithRetries(rmAppRoot, false);
  for (String childNodeName : childNodes) {
    String childNodePath = getNodePath(rmAppRoot, childNodeName);
    byte[] childData = getDataWithRetries(childNodePath, false);
    if (childNodeName.startsWith(ApplicationId.appIdStrPrefix)) {
      // application
      if (LOG.isDebugEnabled()) {
        LOG.debug("Loading application from znode: " + childNodeName);
      }
      ApplicationId appId = ConverterUtils.toApplicationId(childNodeName);
      ApplicationStateDataPBImpl appState =
          new ApplicationStateDataPBImpl(
              ApplicationStateDataProto.parseFrom(childData));
      if (!appId.equals(
          appState.getApplicationSubmissionContext().getApplicationId())) {
        throw new YarnRuntimeException("The child node name is different " +
            "from the application id");
      }
      rmState.appState.put(appId, appState);
      loadApplicationAttemptState(appState, appId);
    } else {
      LOG.info("Unknown child node with name: " + childNodeName);
    }
  }
}
 
Example 6
Source Project: hadoop   Source File: ContainerLogsUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static FileInputStream openLogFileForRead(String containerIdStr, File logFile,
    Context context) throws IOException {
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  ApplicationId applicationId = containerId.getApplicationAttemptId()
      .getApplicationId();
  String user = context.getApplications().get(
      applicationId).getUser();
  
  try {
    return SecureIOUtils.openForRead(logFile, user, null);
  } catch (IOException e) {
    if (e.getMessage().contains(
      "did not match expected owner '" + user
          + "'")) {
      LOG.error(
          "Exception reading log file " + logFile.getAbsolutePath(), e);
      throw new IOException("Exception reading log file. Application submitted by '"
          + user
          + "' doesn't own requested log file : "
          + logFile.getName(), e);
    } else {
      throw new IOException("Exception reading log file. It might be because log "
          + "file was aggregated : " + logFile.getName(), e);
    }
  }
}
 
Example 7
Source Project: hadoop   Source File: NMWebServices.java    License: Apache License 2.0 6 votes vote down vote up
@GET
@Path("/containers/{containerid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public ContainerInfo getNodeContainer(@PathParam("containerid") String id) {
  ContainerId containerId = null;
  init();
  try {
    containerId = ConverterUtils.toContainerId(id);
  } catch (Exception e) {
    throw new BadRequestException("invalid container id, " + id);
  }

  Container container = nmContext.getContainers().get(containerId);
  if (container == null) {
    throw new NotFoundException("container with id, " + id + ", not found");
  }
  return new ContainerInfo(this.nmContext, container, uriInfo.getBaseUri()
      .toString(), webapp.name());

}
 
Example 8
Source Project: big-c   Source File: TestPBRecordImpl.java    License: Apache License 2.0 6 votes vote down vote up
static LocalizerHeartbeatResponse createLocalizerHeartbeatResponse() 
    throws URISyntaxException {
  LocalizerHeartbeatResponse ret =
    recordFactory.newRecordInstance(LocalizerHeartbeatResponse.class);
  assertTrue(ret instanceof LocalizerHeartbeatResponsePBImpl);
  ret.setLocalizerAction(LocalizerAction.LIVE);
  LocalResource rsrc = createResource();
  ArrayList<ResourceLocalizationSpec> rsrcs =
    new ArrayList<ResourceLocalizationSpec>();
  ResourceLocalizationSpec resource =
    recordFactory.newRecordInstance(ResourceLocalizationSpec.class);
  resource.setResource(rsrc);
  resource.setDestinationDirectory(ConverterUtils
    .getYarnUrlFromPath(new Path("/tmp" + System.currentTimeMillis())));
  rsrcs.add(resource);
  ret.setResourceSpecs(rsrcs);
  System.out.println(resource);
  return ret;
}
 
Example 9
Source Project: attic-apex-core   Source File: LoggerUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Makes MDC properties
 */
public static void setupMDC(String service)
{
  MDC.put("apex.service", service);

  String value = StramClientUtils.getHostName();
  MDC.put("apex.node", value == null ? "unknown" : value);

  value = System.getenv(Environment.USER.key());
  if (value != null) {
    MDC.put("apex.user", value);
  }

  value = System.getenv(Environment.CONTAINER_ID.name());
  if (value != null) {
    ContainerId containerId = ConverterUtils.toContainerId(value);
    ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId();
    MDC.put("apex.containerId", containerId.toString());
    MDC.put("apex.applicationId", applicationId.toString());
  }

  value = System.getProperty(APPLICATION_NAME.getLongName());
  if (value != null) {
    MDC.put("apex.application", value);
  }
}
 
Example 10
Source Project: hadoop   Source File: TestNMWebServicesContainers.java    License: Apache License 2.0 6 votes vote down vote up
public void testNodeSingleContainersHelper(String media)
    throws JSONException, Exception {
  WebResource r = resource();
  Application app = new MockApp(1);
  nmContext.getApplications().put(app.getAppId(), app);
  HashMap<String, String> hash = addAppContainers(app);
  Application app2 = new MockApp(2);
  nmContext.getApplications().put(app2.getAppId(), app2);
  addAppContainers(app2);

  for (String id : hash.keySet()) {
    ClientResponse response = r.path("ws").path("v1").path("node")
        .path("containers").path(id).accept(media).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    verifyNodeContainerInfo(json.getJSONObject("container"), nmContext
        .getContainers().get(ConverterUtils.toContainerId(id)));
  }
}
 
Example 11
Source Project: hadoop   Source File: ContainerManagementProtocolProxy.java    License: Apache License 2.0 6 votes vote down vote up
@Private
@VisibleForTesting
protected ContainerManagementProtocol newProxy(final YarnRPC rpc,
    String containerManagerBindAddr, ContainerId containerId, Token token)
    throws InvalidToken {

  if (token == null) {
    throw new InvalidToken("No NMToken sent for "
        + containerManagerBindAddr);
  }
  
  final InetSocketAddress cmAddr =
      NetUtils.createSocketAddr(containerManagerBindAddr);
  LOG.info("Opening proxy : " + containerManagerBindAddr);
  // the user in createRemoteUser in this context has to be ContainerID
  UserGroupInformation user =
      UserGroupInformation.createRemoteUser(containerId
          .getApplicationAttemptId().toString());

  org.apache.hadoop.security.token.Token<NMTokenIdentifier> nmToken =
      ConverterUtils.convertFromYarn(token, cmAddr);
  user.addToken(nmToken);

  return NMProxy.createNMProxy(conf, ContainerManagementProtocol.class,
    user, rpc, cmAddr);
}
 
Example 12
Source Project: incubator-tez   Source File: DagTypeConverters.java    License: Apache License 2.0 6 votes vote down vote up
public static Map<String, LocalResource> createLocalResourceMapFromDAGPlan(
    List<PlanLocalResource> localResourcesList) {
  Map<String, LocalResource> map = new HashMap<String, LocalResource>();
  for(PlanLocalResource res : localResourcesList){
    LocalResource r = new LocalResourcePBImpl();

    //NOTE: have to check every optional field in protobuf generated classes for existence before accessing
    //else we will receive a default value back, eg ""
    if(res.hasPattern()){
      r.setPattern(res.getPattern());
    }
    r.setResource(ConverterUtils.getYarnUrlFromPath(new Path(res.getUri())));  // see above notes on HDFS URL handling
    r.setSize(res.getSize());
    r.setTimestamp(res.getTimeStamp());
    r.setType(DagTypeConverters.convertFromDAGPlan(res.getType()));
    r.setVisibility(DagTypeConverters.convertFromDAGPlan(res.getVisibility()));
    map.put(res.getName(), r);
  }
  return map;
}
 
Example 13
Source Project: big-c   Source File: TestPBRecordImpl.java    License: Apache License 2.0 6 votes vote down vote up
static LocalResourceStatus createLocalResourceStatus() {
  LocalResourceStatus ret =
    recordFactory.newRecordInstance(LocalResourceStatus.class);
  assertTrue(ret instanceof LocalResourceStatusPBImpl);
  ret.setResource(createResource());
  ret.setLocalPath(
      ConverterUtils.getYarnUrlFromPath(
        new Path("file:///local/foo/bar")));
  ret.setStatus(ResourceStatusType.FETCH_SUCCESS);
  ret.setLocalSize(4443L);
  Exception e = new Exception("Dingos.");
  e.setStackTrace(new StackTraceElement[] {
      new StackTraceElement("foo", "bar", "baz", 10),
      new StackTraceElement("sbb", "one", "onm", 10) });
  ret.setException(SerializedException.newInstance(e));
  return ret;
}
 
Example 14
Source Project: reef   Source File: UploaderToJobfolder.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a LocalResource instance for the JAR file referenced by the given Path.
 *
 * @param path
 * @return
 * @throws IOException
 */
LocalResource makeLocalResourceForJarFile(final Path path) throws IOException {
  final LocalResource localResource = Records.newRecord(LocalResource.class);
  final FileStatus status = FileContext.getFileContext(this.fileSystem.getUri()).getFileStatus(path);
  localResource.setType(LocalResourceType.ARCHIVE);
  localResource.setVisibility(LocalResourceVisibility.APPLICATION);
  localResource.setResource(ConverterUtils.getYarnUrlFromPath(status.getPath()));
  localResource.setTimestamp(status.getModificationTime());
  localResource.setSize(status.getLen());
  return localResource;
}
 
Example 15
Source Project: big-c   Source File: ResourceLocalizationService.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Once a container's resources are localized, kill the corresponding
 * {@link ContainerLocalizer}
 */
private void handleContainerResourcesLocalized(
    ContainerLocalizationEvent event) {
  Container c = event.getContainer();
  String locId = ConverterUtils.toString(c.getContainerId());
  localizerTracker.endContainerLocalization(locId);
}
 
Example 16
Source Project: Flink-CEPplus   Source File: FlinkYarnSessionCli.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Nullable
public ApplicationId getClusterId(CommandLine commandLine) {
	if (commandLine.hasOption(applicationId.getOpt())) {
		return ConverterUtils.toApplicationId(commandLine.getOptionValue(applicationId.getOpt()));
	} else if (isYarnPropertiesFileMode(commandLine)) {
		return yarnApplicationIdFromYarnProperties;
	} else {
		return null;
	}
}
 
Example 17
Source Project: big-c   Source File: AMLauncher.java    License: Apache License 2.0 5 votes vote down vote up
protected ContainerManagementProtocol getContainerMgrProxy(
    final ContainerId containerId) {

  final NodeId node = masterContainer.getNodeId();
  final InetSocketAddress containerManagerBindAddress =
      NetUtils.createSocketAddrForHost(node.getHost(), node.getPort());

  final YarnRPC rpc = YarnRPC.create(conf); // TODO: Don't create again and again.

  UserGroupInformation currentUser =
      UserGroupInformation.createRemoteUser(containerId
          .getApplicationAttemptId().toString());

  String user =
      rmContext.getRMApps()
          .get(containerId.getApplicationAttemptId().getApplicationId())
          .getUser();
  org.apache.hadoop.yarn.api.records.Token token =
      rmContext.getNMTokenSecretManager().createNMToken(
          containerId.getApplicationAttemptId(), node, user);
  currentUser.addToken(ConverterUtils.convertFromYarn(token,
      containerManagerBindAddress));

  return currentUser
      .doAs(new PrivilegedAction<ContainerManagementProtocol>() {

        @Override
        public ContainerManagementProtocol run() {
          return (ContainerManagementProtocol) rpc.getProxy(
              ContainerManagementProtocol.class,
              containerManagerBindAddress, conf);
        }
      });
}
 
Example 18
public static void main(String[] args) throws Exception {
  Options options = buildOptions();
  try {
    CommandLine cmd = new DefaultParser().parse(options, args);
    if (!cmd.hasOption(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME) ||
        (!cmd.hasOption(GobblinClusterConfigurationKeys.APPLICATION_ID_OPTION_NAME))) {
      printUsage(options);
      System.exit(1);
    }

    Log4jConfigurationHelper.updateLog4jConfiguration(GobblinApplicationMaster.class,
        GobblinYarnConfigurationKeys.GOBBLIN_YARN_LOG4J_CONFIGURATION_FILE,
        GobblinYarnConfigurationKeys.GOBBLIN_YARN_LOG4J_CONFIGURATION_FILE);

    LOGGER.info(JvmUtils.getJvmInputArguments());

    ContainerId containerId =
        ConverterUtils.toContainerId(System.getenv().get(ApplicationConstants.Environment.CONTAINER_ID.key()));

    try (GobblinApplicationMaster applicationMaster = new GobblinApplicationMaster(
        cmd.getOptionValue(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME),
        cmd.getOptionValue(GobblinClusterConfigurationKeys.APPLICATION_ID_OPTION_NAME), containerId,
        ConfigFactory.load(), new YarnConfiguration())) {

      applicationMaster.start();
    }
  } catch (ParseException pe) {
    printUsage(options);
    System.exit(1);
  }
}
 
Example 19
Source Project: flink   Source File: FlinkYarnSessionCli.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Nullable
public ApplicationId getClusterId(CommandLine commandLine) {
	if (commandLine.hasOption(applicationId.getOpt())) {
		return ConverterUtils.toApplicationId(commandLine.getOptionValue(applicationId.getOpt()));
	} else if (isYarnPropertiesFileMode(commandLine)) {
		return yarnApplicationIdFromYarnProperties;
	} else {
		return null;
	}
}
 
Example 20
Source Project: tez   Source File: DAGSubmittedEvent.java    License: Apache License 2.0 5 votes vote down vote up
public void fromProto(DAGSubmittedProto proto) {
  this.dagID = TezDAGID.fromString(proto.getDagId());
  this.dagPlan = proto.getDagPlan();
  this.dagName = this.dagPlan.getName();
  this.submitTime = proto.getSubmitTime();
  this.applicationAttemptId = ConverterUtils.toApplicationAttemptId(
      proto.getApplicationAttemptId());
  if (proto.hasQueueName()) {
    this.queueName = proto.getQueueName();
  }
  if (proto.hasCumulativeAdditionalAmResources()) {
    this.cumulativeAdditionalLocalResources = DagTypeConverters.convertFromPlanLocalResources(proto
        .getCumulativeAdditionalAmResources());
  }
}
 
Example 21
Source Project: big-c   Source File: LocalResourceRequest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Wrap API resource to match against cache of localized resources.
 * @param resource Resource requested by container
 * @throws URISyntaxException If the path is malformed
 */
public LocalResourceRequest(LocalResource resource)
    throws URISyntaxException {
  this(ConverterUtils.getPathFromYarnURL(resource.getResource()),
      resource.getTimestamp(),
      resource.getType(),
      resource.getVisibility(),
      resource.getPattern());
}
 
Example 22
Source Project: hadoop   Source File: Client.java    License: Apache License 2.0 5 votes vote down vote up
private void addToLocalResources(FileSystem fs, String fileSrcPath,
    String fileDstPath, String appId, Map<String, LocalResource> localResources,
    String resources) throws IOException {
  String suffix =
      appName + "/" + appId + "/" + fileDstPath;
  Path dst =
      new Path(fs.getHomeDirectory(), suffix);
  if (fileSrcPath == null) {
    FSDataOutputStream ostream = null;
    try {
      ostream = FileSystem
          .create(fs, dst, new FsPermission((short) 0710));
      ostream.writeUTF(resources);
    } finally {
      IOUtils.closeQuietly(ostream);
    }
  } else {
    fs.copyFromLocalFile(new Path(fileSrcPath), dst);
  }
  FileStatus scFileStatus = fs.getFileStatus(dst);
  LocalResource scRsrc =
      LocalResource.newInstance(
          ConverterUtils.getYarnUrlFromURI(dst.toUri()),
          LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
          scFileStatus.getLen(), scFileStatus.getModificationTime());
  localResources.put(fileDstPath, scRsrc);
}
 
Example 23
Source Project: garmadon   Source File: ContainerHeader.java    License: Apache License 2.0 5 votes vote down vote up
private Header.SerializedHeader createCachedHeader() {
    String user = System.getenv(ApplicationConstants.Environment.USER.name());
    String containerIdString = System.getenv(ApplicationConstants.Environment.CONTAINER_ID.name());
    String host = System.getenv(ApplicationConstants.Environment.NM_HOST.name());

    // Get applicationID
    ContainerId containerId = ConverterUtils.toContainerId(containerIdString);
    ApplicationAttemptId appAttemptID = containerId.getApplicationAttemptId();
    ApplicationId appId = appAttemptID.getApplicationId();

    //build the header for the whole application once
    return Header.newBuilder()
            .withId(appId.toString())
            .addTag(Header.Tag.YARN_APPLICATION.name())
            .addTags(System.getProperty("garmadon.tags"))
            .withHostname(host)
            .withApplicationID(appId.toString())
            .withAttemptID(appAttemptID.toString())
            .withUser(user)
            .withContainerID(containerIdString)
            .withPid(HeaderUtils.getPid())
            .withFramework(framework.toString())
            .withComponent(component.name())
            .withExecutorId(executorId)
            .withMainClass(mainClass)
            .buildSerializedHeader();
}
 
Example 24
Source Project: samza   Source File: TestYarnContainerHeartbeatServlet.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testContainerHeartbeatWhenValid()
    throws IOException {
  String validContainerId = "container_1350670447861_0003_01_000002";
  when(container.id()).thenReturn(ConverterUtils.toContainerId(validContainerId));
  yarnAppState.runningProcessors.put(validContainerId, container);
  URL url = new URL(webApp.getUrl().toString() + "containerHeartbeat?executionContainerId=" + validContainerId);
  String response = HttpUtil.read(url, 1000, new ExponentialSleepStrategy());
  heartbeat = mapper.readValue(response, ContainerHeartbeatResponse.class);
  Assert.assertTrue(heartbeat.isAlive());
}
 
Example 25
Source Project: big-c   Source File: TestFSRMStateStore.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void testFSRMStateStore() throws Exception {
  HdfsConfiguration conf = new HdfsConfiguration();
  MiniDFSCluster cluster =
          new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  try {
    fsTester = new TestFSRMStateStoreTester(cluster, false);
    // If the state store is FileSystemRMStateStore then add corrupted entry.
    // It should discard the entry and remove it from file system.
    FSDataOutputStream fsOut = null;
    FileSystemRMStateStore fileSystemRMStateStore =
            (FileSystemRMStateStore) fsTester.getRMStateStore();
    String appAttemptIdStr3 = "appattempt_1352994193343_0001_000003";
    ApplicationAttemptId attemptId3 =
            ConverterUtils.toApplicationAttemptId(appAttemptIdStr3);
    Path appDir =
            fsTester.store.getAppDir(attemptId3.getApplicationId().toString());
    Path tempAppAttemptFile =
            new Path(appDir, attemptId3.toString() + ".tmp");
    fsOut = fileSystemRMStateStore.fs.create(tempAppAttemptFile, false);
    fsOut.write("Some random data ".getBytes());
    fsOut.close();

    testRMAppStateStore(fsTester);
    Assert.assertFalse(fsTester.workingDirPathURI
            .getFileSystem(conf).exists(tempAppAttemptFile));
    testRMDTSecretManagerStateStore(fsTester);
    testCheckVersion(fsTester);
    testEpoch(fsTester);
    testAppDeletion(fsTester);
    testDeleteStore(fsTester);
    testAMRMTokenSecretManagerStateStore(fsTester);
  } finally {
    cluster.shutdown();
  }
}
 
Example 26
Source Project: PoseidonX   Source File: YarnClientProxy.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * 根据APPID获得ApplicationReport
 * @param applicationId
 * @throws Exception
 */
public static ApplicationReport getApplicationReportByAppId(String applicationId) {
    if(StringUtils.isBlank(applicationId)){
        return null;
    }
    ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
    ApplicationReport report = null;
    try {
        report = yarnClient.getApplicationReport(appId);
    } catch (Exception e) {
        //LOGGER.error("YarnClientProxy getApplicationReportByAppId is error",e);
    }
    return report;
}
 
Example 27
Source Project: spydra   Source File: HistoryLogUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
  String clientId = args[0];
  String appIdStr = args[1];
  String bucket = args[2];

  ApplicationId appId = ConverterUtils.toApplicationId(appIdStr);
  Configuration cfg = generateHadoopConfig(clientId, DEFAULT_USER_NAME, bucket);

  logger.info("Dumping log files");
  dumpFullLogs(cfg, appId);
  logger.info("Dumping full history information");
  dumpFullHistory(cfg, appId);
  logger.info("Starting local JHS");
  startJhs(cfg);
}
 
Example 28
Source Project: big-c   Source File: NMController.java    License: Apache License 2.0 5 votes vote down vote up
public void logs() {
  String containerIdStr = $(CONTAINER_ID);
  ContainerId containerId = null;
  try {
    containerId = ConverterUtils.toContainerId(containerIdStr);
  } catch (IllegalArgumentException e) {
    render(ContainerLogsPage.class);
    return;
  }
  ApplicationId appId =
      containerId.getApplicationAttemptId().getApplicationId();
  Application app = nmContext.getApplications().get(appId);
  if (app == null
      && nmConf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
          YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED)) {
    String logServerUrl = nmConf.get(YarnConfiguration.YARN_LOG_SERVER_URL);
    String redirectUrl = null;
    if (logServerUrl == null || logServerUrl.isEmpty()) {
      redirectUrl = "false";
    } else {
      redirectUrl =
          url(logServerUrl, nmContext.getNodeId().toString(), containerIdStr,
              containerIdStr, $(APP_OWNER));
    }
    set(ContainerLogsPage.REDIRECT_URL, redirectUrl);
  }
  render(ContainerLogsPage.class);
}
 
Example 29
Source Project: Hi-WAY   Source File: Data.java    License: Apache License 2.0 5 votes vote down vote up
public void addToLocalResourceMap(Map<String, LocalResource> localResources) throws IOException {
	Path dest = getHdfsPath();

	LocalResource rsrc = Records.newRecord(LocalResource.class);
	rsrc.setType(LocalResourceType.FILE);
	rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
	rsrc.setResource(ConverterUtils.getYarnUrlFromPath(dest));

	FileStatus status = hdfs.getFileStatus(dest);
	rsrc.setTimestamp(status.getModificationTime());
	rsrc.setSize(status.getLen());

	localResources.put(getLocalPath().toString(), rsrc);
}
 
Example 30
Source Project: big-c   Source File: RMWebServices.java    License: Apache License 2.0 5 votes vote down vote up
@GET
@Path("/nodes/{nodeId}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public NodeInfo getNode(@PathParam("nodeId") String nodeId) {
  init();
  if (nodeId == null || nodeId.isEmpty()) {
    throw new NotFoundException("nodeId, " + nodeId + ", is empty or null");
  }
  ResourceScheduler sched = this.rm.getResourceScheduler();
  if (sched == null) {
    throw new NotFoundException("Null ResourceScheduler instance");
  }
  NodeId nid = ConverterUtils.toNodeId(nodeId);
  RMNode ni = this.rm.getRMContext().getRMNodes().get(nid);
  boolean isInactive = false;
  if (ni == null) {
    ni = this.rm.getRMContext().getInactiveRMNodes().get(nid.getHost());
    if (ni == null) {
      throw new NotFoundException("nodeId, " + nodeId + ", is not found");
    }
    isInactive = true;
  }
  NodeInfo nodeInfo = new NodeInfo(ni, sched);
  if (isInactive) {
    nodeInfo.setNodeHTTPAddress(EMPTY);
  }
  return nodeInfo;
}