org.apache.hadoop.yarn.util.ConverterUtils Java Examples

The following examples show how to use org.apache.hadoop.yarn.util.ConverterUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestPBRecordImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
static LocalResourceStatus createLocalResourceStatus() {
  LocalResourceStatus ret =
    recordFactory.newRecordInstance(LocalResourceStatus.class);
  assertTrue(ret instanceof LocalResourceStatusPBImpl);
  ret.setResource(createResource());
  ret.setLocalPath(
      ConverterUtils.getYarnUrlFromPath(
        new Path("file:///local/foo/bar")));
  ret.setStatus(ResourceStatusType.FETCH_SUCCESS);
  ret.setLocalSize(4443L);
  Exception e = new Exception("Dingos.");
  e.setStackTrace(new StackTraceElement[] {
      new StackTraceElement("foo", "bar", "baz", 10),
      new StackTraceElement("sbb", "one", "onm", 10) });
  ret.setException(SerializedException.newInstance(e));
  return ret;
}
 
Example #2
Source File: RMWebServices.java    From big-c with Apache License 2.0 6 votes vote down vote up
private RMApp getRMAppForAppId(String appId) {

    if (appId == null || appId.isEmpty()) {
      throw new NotFoundException("appId, " + appId + ", is empty or null");
    }
    ApplicationId id;
    try {
      id = ConverterUtils.toApplicationId(recordFactory, appId);
    } catch (NumberFormatException e) {
      throw new NotFoundException("appId is invalid");
    }
    if (id == null) {
      throw new NotFoundException("appId is invalid");
    }
    RMApp app = rm.getRMContext().getRMApps().get(id);
    if (app == null) {
      throw new NotFoundException("app with id: " + appId + " not found");
    }
    return app;
  }
 
Example #3
Source File: TestMRAppMaster.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testMRAppMasterForDifferentUser() throws IOException,
    InterruptedException {
  String applicationAttemptIdStr = "appattempt_1317529182569_0004_000001";
  String containerIdStr = "container_1317529182569_0004_000001_1";
  
  String userName = "TestAppMasterUser";
  ApplicationAttemptId applicationAttemptId = ConverterUtils
      .toApplicationAttemptId(applicationAttemptIdStr);
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  MRAppMasterTest appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
          System.currentTimeMillis());
  JobConf conf = new JobConf();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
  Path userPath = new Path(stagingDir, userName);
  Path userStagingPath = new Path(userPath, ".staging");
  assertEquals(userStagingPath.toString(),
    appMaster.stagingDirPath.toString());
}
 
Example #4
Source File: ContainerManagementProtocolProxy.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Private
@VisibleForTesting
protected ContainerManagementProtocol newProxy(final YarnRPC rpc,
    String containerManagerBindAddr, ContainerId containerId, Token token)
    throws InvalidToken {

  if (token == null) {
    throw new InvalidToken("No NMToken sent for "
        + containerManagerBindAddr);
  }
  
  final InetSocketAddress cmAddr =
      NetUtils.createSocketAddr(containerManagerBindAddr);
  LOG.info("Opening proxy : " + containerManagerBindAddr);
  // the user in createRemoteUser in this context has to be ContainerID
  UserGroupInformation user =
      UserGroupInformation.createRemoteUser(containerId
          .getApplicationAttemptId().toString());

  org.apache.hadoop.security.token.Token<NMTokenIdentifier> nmToken =
      ConverterUtils.convertFromYarn(token, cmAddr);
  user.addToken(nmToken);

  return NMProxy.createNMProxy(conf, ContainerManagementProtocol.class,
    user, rpc, cmAddr);
}
 
Example #5
Source File: AggregatedLogsBlock.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private NodeId verifyAndGetNodeId(Block html) {
  String nodeIdStr = $(NM_NODENAME);
  if (nodeIdStr == null || nodeIdStr.isEmpty()) {
    html.h1()._("Cannot get container logs without a NodeId")._();
    return null;
  }
  NodeId nodeId = null;
  try {
    nodeId = ConverterUtils.toNodeId(nodeIdStr);
  } catch (IllegalArgumentException e) {
    html.h1()._("Cannot get container logs. Invalid nodeId: " + nodeIdStr)
        ._();
    return null;
  }
  return nodeId;
}
 
Example #6
Source File: ZKRMStateStore.java    From big-c with Apache License 2.0 6 votes vote down vote up
private synchronized void loadRMAppState(RMState rmState) throws Exception {
  List<String> childNodes = getChildrenWithRetries(rmAppRoot, false);
  for (String childNodeName : childNodes) {
    String childNodePath = getNodePath(rmAppRoot, childNodeName);
    byte[] childData = getDataWithRetries(childNodePath, false);
    if (childNodeName.startsWith(ApplicationId.appIdStrPrefix)) {
      // application
      if (LOG.isDebugEnabled()) {
        LOG.debug("Loading application from znode: " + childNodeName);
      }
      ApplicationId appId = ConverterUtils.toApplicationId(childNodeName);
      ApplicationStateDataPBImpl appState =
          new ApplicationStateDataPBImpl(
              ApplicationStateDataProto.parseFrom(childData));
      if (!appId.equals(
          appState.getApplicationSubmissionContext().getApplicationId())) {
        throw new YarnRuntimeException("The child node name is different " +
            "from the application id");
      }
      rmState.appState.put(appId, appState);
      loadApplicationAttemptState(appState, appId);
    } else {
      LOG.info("Unknown child node with name: " + childNodeName);
    }
  }
}
 
Example #7
Source File: ContainerLogsUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public static FileInputStream openLogFileForRead(String containerIdStr, File logFile,
    Context context) throws IOException {
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  ApplicationId applicationId = containerId.getApplicationAttemptId()
      .getApplicationId();
  String user = context.getApplications().get(
      applicationId).getUser();
  
  try {
    return SecureIOUtils.openForRead(logFile, user, null);
  } catch (IOException e) {
    if (e.getMessage().contains(
      "did not match expected owner '" + user
          + "'")) {
      LOG.error(
          "Exception reading log file " + logFile.getAbsolutePath(), e);
      throw new IOException("Exception reading log file. Application submitted by '"
          + user
          + "' doesn't own requested log file : "
          + logFile.getName(), e);
    } else {
      throw new IOException("Exception reading log file. It might be because log "
          + "file was aggregated : " + logFile.getName(), e);
    }
  }
}
 
Example #8
Source File: NMWebServices.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@GET
@Path("/containers/{containerid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public ContainerInfo getNodeContainer(@PathParam("containerid") String id) {
  ContainerId containerId = null;
  init();
  try {
    containerId = ConverterUtils.toContainerId(id);
  } catch (Exception e) {
    throw new BadRequestException("invalid container id, " + id);
  }

  Container container = nmContext.getContainers().get(containerId);
  if (container == null) {
    throw new NotFoundException("container with id, " + id + ", not found");
  }
  return new ContainerInfo(this.nmContext, container, uriInfo.getBaseUri()
      .toString(), webapp.name());

}
 
Example #9
Source File: TestPBRecordImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
static LocalizerHeartbeatResponse createLocalizerHeartbeatResponse() 
    throws URISyntaxException {
  LocalizerHeartbeatResponse ret =
    recordFactory.newRecordInstance(LocalizerHeartbeatResponse.class);
  assertTrue(ret instanceof LocalizerHeartbeatResponsePBImpl);
  ret.setLocalizerAction(LocalizerAction.LIVE);
  LocalResource rsrc = createResource();
  ArrayList<ResourceLocalizationSpec> rsrcs =
    new ArrayList<ResourceLocalizationSpec>();
  ResourceLocalizationSpec resource =
    recordFactory.newRecordInstance(ResourceLocalizationSpec.class);
  resource.setResource(rsrc);
  resource.setDestinationDirectory(ConverterUtils
    .getYarnUrlFromPath(new Path("/tmp" + System.currentTimeMillis())));
  rsrcs.add(resource);
  ret.setResourceSpecs(rsrcs);
  System.out.println(resource);
  return ret;
}
 
Example #10
Source File: LoggerUtil.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
/**
 * Makes MDC properties
 */
public static void setupMDC(String service)
{
  MDC.put("apex.service", service);

  String value = StramClientUtils.getHostName();
  MDC.put("apex.node", value == null ? "unknown" : value);

  value = System.getenv(Environment.USER.key());
  if (value != null) {
    MDC.put("apex.user", value);
  }

  value = System.getenv(Environment.CONTAINER_ID.name());
  if (value != null) {
    ContainerId containerId = ConverterUtils.toContainerId(value);
    ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId();
    MDC.put("apex.containerId", containerId.toString());
    MDC.put("apex.applicationId", applicationId.toString());
  }

  value = System.getProperty(APPLICATION_NAME.getLongName());
  if (value != null) {
    MDC.put("apex.application", value);
  }
}
 
Example #11
Source File: TestNMWebServicesContainers.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public void testNodeSingleContainersHelper(String media)
    throws JSONException, Exception {
  WebResource r = resource();
  Application app = new MockApp(1);
  nmContext.getApplications().put(app.getAppId(), app);
  HashMap<String, String> hash = addAppContainers(app);
  Application app2 = new MockApp(2);
  nmContext.getApplications().put(app2.getAppId(), app2);
  addAppContainers(app2);

  for (String id : hash.keySet()) {
    ClientResponse response = r.path("ws").path("v1").path("node")
        .path("containers").path(id).accept(media).get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    JSONObject json = response.getEntity(JSONObject.class);
    verifyNodeContainerInfo(json.getJSONObject("container"), nmContext
        .getContainers().get(ConverterUtils.toContainerId(id)));
  }
}
 
Example #12
Source File: ContainerManagementProtocolProxy.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Private
@VisibleForTesting
protected ContainerManagementProtocol newProxy(final YarnRPC rpc,
    String containerManagerBindAddr, ContainerId containerId, Token token)
    throws InvalidToken {

  if (token == null) {
    throw new InvalidToken("No NMToken sent for "
        + containerManagerBindAddr);
  }
  
  final InetSocketAddress cmAddr =
      NetUtils.createSocketAddr(containerManagerBindAddr);
  LOG.info("Opening proxy : " + containerManagerBindAddr);
  // the user in createRemoteUser in this context has to be ContainerID
  UserGroupInformation user =
      UserGroupInformation.createRemoteUser(containerId
          .getApplicationAttemptId().toString());

  org.apache.hadoop.security.token.Token<NMTokenIdentifier> nmToken =
      ConverterUtils.convertFromYarn(token, cmAddr);
  user.addToken(nmToken);

  return NMProxy.createNMProxy(conf, ContainerManagementProtocol.class,
    user, rpc, cmAddr);
}
 
Example #13
Source File: DagTypeConverters.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
public static Map<String, LocalResource> createLocalResourceMapFromDAGPlan(
    List<PlanLocalResource> localResourcesList) {
  Map<String, LocalResource> map = new HashMap<String, LocalResource>();
  for(PlanLocalResource res : localResourcesList){
    LocalResource r = new LocalResourcePBImpl();

    //NOTE: have to check every optional field in protobuf generated classes for existence before accessing
    //else we will receive a default value back, eg ""
    if(res.hasPattern()){
      r.setPattern(res.getPattern());
    }
    r.setResource(ConverterUtils.getYarnUrlFromPath(new Path(res.getUri())));  // see above notes on HDFS URL handling
    r.setSize(res.getSize());
    r.setTimestamp(res.getTimeStamp());
    r.setType(DagTypeConverters.convertFromDAGPlan(res.getType()));
    r.setVisibility(DagTypeConverters.convertFromDAGPlan(res.getVisibility()));
    map.put(res.getName(), r);
  }
  return map;
}
 
Example #14
Source File: UploaderToJobfolder.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a LocalResource instance for the JAR file referenced by the given Path.
 *
 * @param path
 * @return
 * @throws IOException
 */
LocalResource makeLocalResourceForJarFile(final Path path) throws IOException {
  final LocalResource localResource = Records.newRecord(LocalResource.class);
  final FileStatus status = FileContext.getFileContext(this.fileSystem.getUri()).getFileStatus(path);
  localResource.setType(LocalResourceType.ARCHIVE);
  localResource.setVisibility(LocalResourceVisibility.APPLICATION);
  localResource.setResource(ConverterUtils.getYarnUrlFromPath(status.getPath()));
  localResource.setTimestamp(status.getModificationTime());
  localResource.setSize(status.getLen());
  return localResource;
}
 
Example #15
Source File: RMWebServices.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Create the actual ApplicationSubmissionContext to be submitted to the RM
 * from the information provided by the user.
 * 
 * @param newApp
 *          the information provided by the user
 * @return returns the constructed ApplicationSubmissionContext
 * @throws IOException
 */
protected ApplicationSubmissionContext createAppSubmissionContext(
    ApplicationSubmissionContextInfo newApp) throws IOException {

  // create local resources and app submission context

  ApplicationId appid;
  String error =
      "Could not parse application id " + newApp.getApplicationId();
  try {
    appid =
        ConverterUtils.toApplicationId(recordFactory,
          newApp.getApplicationId());
  } catch (Exception e) {
    throw new BadRequestException(error);
  }
  ApplicationSubmissionContext appContext =
      ApplicationSubmissionContext.newInstance(appid,
        newApp.getApplicationName(), newApp.getQueue(),
        Priority.newInstance(newApp.getPriority()),
        createContainerLaunchContext(newApp), newApp.getUnmanagedAM(),
        newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(),
        createAppSubmissionContextResource(newApp),
        newApp.getApplicationType(),
        newApp.getKeepContainersAcrossApplicationAttempts(),
        newApp.getAppNodeLabelExpression(),
        newApp.getAMContainerNodeLabelExpression());
  appContext.setApplicationTags(newApp.getApplicationTags());

  return appContext;
}
 
Example #16
Source File: TestZKRMStateStore.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testDuplicateRMAppDeletion() throws Exception {
  TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester();
  long submitTime = System.currentTimeMillis();
  long startTime = System.currentTimeMillis() + 1234;
  RMStateStore store = zkTester.getRMStateStore();
  TestDispatcher dispatcher = new TestDispatcher();
  store.setRMDispatcher(dispatcher);

  ApplicationAttemptId attemptIdRemoved = ConverterUtils
      .toApplicationAttemptId("appattempt_1352994193343_0002_000001");
  ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
  storeApp(store, appIdRemoved, submitTime, startTime);
  storeAttempt(store, attemptIdRemoved,
      "container_1352994193343_0002_01_000001", null, null, dispatcher);

  ApplicationSubmissionContext context =
      new ApplicationSubmissionContextPBImpl();
  context.setApplicationId(appIdRemoved);
  ApplicationStateData appStateRemoved =
      ApplicationStateData.newInstance(
          submitTime, startTime, context, "user1");
  appStateRemoved.attempts.put(attemptIdRemoved, null);
  store.removeApplicationStateInternal(appStateRemoved);
  try {
    store.removeApplicationStateInternal(appStateRemoved);
  } catch (KeeperException.NoNodeException nne) {
    Assert.fail("NoNodeException should not happen.");
  }
  store.close();
}
 
Example #17
Source File: ContainerPage.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected void render(Block html) {
  ContainerId containerID;
  try {
    containerID = ConverterUtils.toContainerId($(CONTAINER_ID));
  } catch (IllegalArgumentException e) {
    html.p()._("Invalid containerId " + $(CONTAINER_ID))._();
    return;
  }

  DIV<Hamlet> div = html.div("#content");
  Container container = this.nmContext.getContainers().get(containerID);
  if (container == null) {
    div.h1("Unknown Container. Container might have completed, "
            + "please go back to the previous page and retry.")._();
    return;
  }
  ContainerInfo info = new ContainerInfo(this.nmContext, container);

  info("Container information")
    ._("ContainerID", info.getId())
    ._("ContainerState", info.getState())
    ._("ExitStatus", info.getExitStatus())
    ._("Diagnostics", info.getDiagnostics())
    ._("User", info.getUser())
    ._("TotalMemoryNeeded", info.getMemoryNeeded())
    ._("TotalVCoresNeeded", info.getVCoresNeeded())
    ._("logs", info.getShortLogLink(), "Link to logs");
  html._(InfoBlock.class);
}
 
Example #18
Source File: NodeManagerBuilderUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static ResourceLocalizationSpec newResourceLocalizationSpec(
    LocalResource rsrc, Path path) {
  URL local = ConverterUtils.getYarnUrlFromPath(path);
  ResourceLocalizationSpec resourceLocalizationSpec =
      Records.newRecord(ResourceLocalizationSpec.class);
  resourceLocalizationSpec.setDestinationDirectory(local);
  resourceLocalizationSpec.setResource(rsrc);
  return resourceLocalizationSpec;
}
 
Example #19
Source File: ContainerImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public String toString() {
  this.readLock.lock();
  try {
    return ConverterUtils.toString(this.containerId);
  } finally {
    this.readLock.unlock();
  }
}
 
Example #20
Source File: ContainerLaunch.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Loop through for a time-bounded interval waiting to
 * read the process id from a file generated by a running process.
 * @param pidFilePath File from which to read the process id
 * @return Process ID
 * @throws Exception
 */
private String getContainerPid(Path pidFilePath) throws Exception {
  String containerIdStr = 
      ConverterUtils.toString(container.getContainerId());
  String processId = null;
  LOG.debug("Accessing pid for container " + containerIdStr
      + " from pid file " + pidFilePath);
  int sleepCounter = 0;
  final int sleepInterval = 100;

  // loop waiting for pid file to show up 
  // until our timer expires in which case we admit defeat
  while (true) {
    processId = ProcessIdFileReader.getProcessId(pidFilePath);
    if (processId != null) {
      LOG.debug("Got pid " + processId + " for container "
          + containerIdStr);
      break;
    }
    else if ((sleepCounter*sleepInterval) > maxKillWaitTime) {
      LOG.info("Could not get pid for " + containerIdStr
      		+ ". Waited for " + maxKillWaitTime + " ms.");
      break;
    }
    else {
      ++sleepCounter;
      Thread.sleep(sleepInterval);
    }
  }
  return processId;
}
 
Example #21
Source File: LocalizerResourceRequestEvent.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public LocalizerResourceRequestEvent(LocalizedResource resource,
    LocalResourceVisibility vis, LocalizerContext context, String pattern) {
  super(LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION,
      ConverterUtils.toString(context.getContainerId()));
  this.vis = vis;
  this.context = context;
  this.resource = resource;
  this.pattern = pattern;
}
 
Example #22
Source File: AMLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected ContainerManagementProtocol getContainerMgrProxy(
    final ContainerId containerId) {

  final NodeId node = masterContainer.getNodeId();
  final InetSocketAddress containerManagerBindAddress =
      NetUtils.createSocketAddrForHost(node.getHost(), node.getPort());

  final YarnRPC rpc = YarnRPC.create(conf); // TODO: Don't create again and again.

  UserGroupInformation currentUser =
      UserGroupInformation.createRemoteUser(containerId
          .getApplicationAttemptId().toString());

  String user =
      rmContext.getRMApps()
          .get(containerId.getApplicationAttemptId().getApplicationId())
          .getUser();
  org.apache.hadoop.yarn.api.records.Token token =
      rmContext.getNMTokenSecretManager().createNMToken(
          containerId.getApplicationAttemptId(), node, user);
  currentUser.addToken(ConverterUtils.convertFromYarn(token,
      containerManagerBindAddress));

  return currentUser
      .doAs(new PrivilegedAction<ContainerManagementProtocol>() {

        @Override
        public ContainerManagementProtocol run() {
          return (ContainerManagementProtocol) rpc.getProxy(
              ContainerManagementProtocol.class,
              containerManagerBindAddress, conf);
        }
      });
}
 
Example #23
Source File: LocalResourceRequest.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Wrap API resource to match against cache of localized resources.
 * @param resource Resource requested by container
 * @throws URISyntaxException If the path is malformed
 */
public LocalResourceRequest(LocalResource resource)
    throws URISyntaxException {
  this(ConverterUtils.getPathFromYarnURL(resource.getResource()),
      resource.getTimestamp(),
      resource.getType(),
      resource.getVisibility(),
      resource.getPattern());
}
 
Example #24
Source File: ResourceLocalizationService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
LocalResourcesTracker getLocalResourcesTracker(
    LocalResourceVisibility visibility, String user, ApplicationId appId) {
  switch (visibility) {
    default:
    case PUBLIC:
      return publicRsrc;
    case PRIVATE:
      return privateRsrc.get(user);
    case APPLICATION:
      return appRsrc.get(ConverterUtils.toString(appId));
  }
}
 
Example #25
Source File: TestAMWebServicesAttempts.java    From big-c with Apache License 2.0 5 votes vote down vote up
public void verifyTaskAttemptGeneric(TaskAttempt ta, TaskType ttype,
    String id, String state, String type, String rack,
    String nodeHttpAddress, String diagnostics, String assignedContainerId,
    long startTime, long finishTime, long elapsedTime, float progress) {

  TaskAttemptId attid = ta.getID();
  String attemptId = MRApps.toString(attid);

  WebServicesTestUtils.checkStringMatch("id", attemptId, id);
  WebServicesTestUtils.checkStringMatch("type", ttype.toString(), type);
  WebServicesTestUtils.checkStringMatch("state", ta.getState().toString(),
      state);
  WebServicesTestUtils.checkStringMatch("rack", ta.getNodeRackName(), rack);
  WebServicesTestUtils.checkStringMatch("nodeHttpAddress",
      ta.getNodeHttpAddress(), nodeHttpAddress);

  String expectDiag = "";
  List<String> diagnosticsList = ta.getDiagnostics();
  if (diagnosticsList != null && !diagnostics.isEmpty()) {
    StringBuffer b = new StringBuffer();
    for (String diag : diagnosticsList) {
      b.append(diag);
    }
    expectDiag = b.toString();
  }
  WebServicesTestUtils.checkStringMatch("diagnostics", expectDiag,
      diagnostics);
  WebServicesTestUtils.checkStringMatch("assignedContainerId",
      ConverterUtils.toString(ta.getAssignedContainerID()),
      assignedContainerId);

  assertEquals("startTime wrong", ta.getLaunchTime(), startTime);
  assertEquals("finishTime wrong", ta.getFinishTime(), finishTime);
  assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime);
  assertEquals("progress wrong", ta.getProgress() * 100, progress, 1e-3f);
}
 
Example #26
Source File: TestHsWebServicesAttempts.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public void verifyTaskAttemptGeneric(TaskAttempt ta, TaskType ttype,
    String id, String state, String type, String rack,
    String nodeHttpAddress, String diagnostics, String assignedContainerId,
    long startTime, long finishTime, long elapsedTime, float progress) {

  TaskAttemptId attid = ta.getID();
  String attemptId = MRApps.toString(attid);

  WebServicesTestUtils.checkStringMatch("id", attemptId, id);
  WebServicesTestUtils.checkStringMatch("type", ttype.toString(), type);
  WebServicesTestUtils.checkStringMatch("state", ta.getState().toString(),
      state);
  WebServicesTestUtils.checkStringMatch("rack", ta.getNodeRackName(), rack);
  WebServicesTestUtils.checkStringMatch("nodeHttpAddress",
      ta.getNodeHttpAddress(), nodeHttpAddress);

  String expectDiag = "";
  List<String> diagnosticsList = ta.getDiagnostics();
  if (diagnosticsList != null && !diagnostics.isEmpty()) {
    StringBuffer b = new StringBuffer();
    for (String diag : diagnosticsList) {
      b.append(diag);
    }
    expectDiag = b.toString();
  }
  WebServicesTestUtils.checkStringMatch("diagnostics", expectDiag,
      diagnostics);
  WebServicesTestUtils.checkStringMatch("assignedContainerId",
      ConverterUtils.toString(ta.getAssignedContainerID()),
      assignedContainerId);

  assertEquals("startTime wrong", ta.getLaunchTime(), startTime);
  assertEquals("finishTime wrong", ta.getFinishTime(), finishTime);
  assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime);
  assertEquals("progress wrong", ta.getProgress() * 100, progress, 1e-3f);
}
 
Example #27
Source File: GobblinApplicationMaster.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
  Options options = buildOptions();
  try {
    CommandLine cmd = new DefaultParser().parse(options, args);
    if (!cmd.hasOption(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME) ||
        (!cmd.hasOption(GobblinClusterConfigurationKeys.APPLICATION_ID_OPTION_NAME))) {
      printUsage(options);
      System.exit(1);
    }

    Log4jConfigurationHelper.updateLog4jConfiguration(GobblinApplicationMaster.class,
        GobblinYarnConfigurationKeys.GOBBLIN_YARN_LOG4J_CONFIGURATION_FILE,
        GobblinYarnConfigurationKeys.GOBBLIN_YARN_LOG4J_CONFIGURATION_FILE);

    LOGGER.info(JvmUtils.getJvmInputArguments());

    ContainerId containerId =
        ConverterUtils.toContainerId(System.getenv().get(ApplicationConstants.Environment.CONTAINER_ID.key()));

    try (GobblinApplicationMaster applicationMaster = new GobblinApplicationMaster(
        cmd.getOptionValue(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME),
        cmd.getOptionValue(GobblinClusterConfigurationKeys.APPLICATION_ID_OPTION_NAME), containerId,
        ConfigFactory.load(), new YarnConfiguration())) {

      applicationMaster.start();
    }
  } catch (ParseException pe) {
    printUsage(options);
    System.exit(1);
  }
}
 
Example #28
Source File: YARNRunner.java    From big-c with Apache License 2.0 5 votes vote down vote up
private LocalResource createApplicationResource(FileContext fs, Path p, LocalResourceType type)
    throws IOException {
  LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class);
  FileStatus rsrcStat = fs.getFileStatus(p);
  rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs
      .getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
  rsrc.setSize(rsrcStat.getLen());
  rsrc.setTimestamp(rsrcStat.getModificationTime());
  rsrc.setType(type);
  rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
  return rsrc;
}
 
Example #29
Source File: ContainerPage.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
protected void render(Block html) {
  ContainerId containerID;
  try {
    containerID = ConverterUtils.toContainerId($(CONTAINER_ID));
  } catch (IllegalArgumentException e) {
    html.p()._("Invalid containerId " + $(CONTAINER_ID))._();
    return;
  }

  DIV<Hamlet> div = html.div("#content");
  Container container = this.nmContext.getContainers().get(containerID);
  if (container == null) {
    div.h1("Unknown Container. Container might have completed, "
            + "please go back to the previous page and retry.")._();
    return;
  }
  ContainerInfo info = new ContainerInfo(this.nmContext, container);

  info("Container information")
    ._("ContainerID", info.getId())
    ._("ContainerState", info.getState())
    ._("ExitStatus", info.getExitStatus())
    ._("Diagnostics", info.getDiagnostics())
    ._("User", info.getUser())
    ._("TotalMemoryNeeded", info.getMemoryNeeded())
    ._("TotalVCoresNeeded", info.getVCoresNeeded())
    ._("logs", info.getShortLogLink(), "Link to logs");
  html._(InfoBlock.class);
}
 
Example #30
Source File: NMLeveldbStateStoreService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static void loadContainerToken(RecoveredContainerTokensState state,
    String key, String containerIdStr, byte[] value) throws IOException {
  ContainerId containerId;
  Long expTime;
  try {
    containerId = ConverterUtils.toContainerId(containerIdStr);
    expTime = Long.parseLong(asString(value));
  } catch (IllegalArgumentException e) {
    throw new IOException("Bad container token state for " + key, e);
  }
  state.activeTokens.put(containerId, expTime);
}