Java Code Examples for org.apache.hadoop.yarn.util.ConverterUtils#toContainerId()

The following examples show how to use org.apache.hadoop.yarn.util.ConverterUtils#toContainerId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestMRAppMaster.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testMRAppMasterForDifferentUser() throws IOException,
    InterruptedException {
  String applicationAttemptIdStr = "appattempt_1317529182569_0004_000001";
  String containerIdStr = "container_1317529182569_0004_000001_1";
  
  String userName = "TestAppMasterUser";
  ApplicationAttemptId applicationAttemptId = ConverterUtils
      .toApplicationAttemptId(applicationAttemptIdStr);
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  MRAppMasterTest appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
          System.currentTimeMillis());
  JobConf conf = new JobConf();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
  Path userPath = new Path(stagingDir, userName);
  Path userStagingPath = new Path(userPath, ".staging");
  assertEquals(userStagingPath.toString(),
    appMaster.stagingDirPath.toString());
}
 
Example 2
Source File: TestMRAppMaster.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testMRAppMasterForDifferentUser() throws IOException,
    InterruptedException {
  String applicationAttemptIdStr = "appattempt_1317529182569_0004_000001";
  String containerIdStr = "container_1317529182569_0004_000001_1";
  
  String userName = "TestAppMasterUser";
  ApplicationAttemptId applicationAttemptId = ConverterUtils
      .toApplicationAttemptId(applicationAttemptIdStr);
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  MRAppMasterTest appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
          System.currentTimeMillis());
  JobConf conf = new JobConf();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
  Path userPath = new Path(stagingDir, userName);
  Path userStagingPath = new Path(userPath, ".staging");
  assertEquals(userStagingPath.toString(),
    appMaster.stagingDirPath.toString());
}
 
Example 3
Source File: ContainerLogsUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up
public static FileInputStream openLogFileForRead(String containerIdStr, File logFile,
    Context context) throws IOException {
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  ApplicationId applicationId = containerId.getApplicationAttemptId()
      .getApplicationId();
  String user = context.getApplications().get(
      applicationId).getUser();
  
  try {
    return SecureIOUtils.openForRead(logFile, user, null);
  } catch (IOException e) {
    if (e.getMessage().contains(
      "did not match expected owner '" + user
          + "'")) {
      LOG.error(
          "Exception reading log file " + logFile.getAbsolutePath(), e);
      throw new IOException("Exception reading log file. Application submitted by '"
          + user
          + "' doesn't own requested log file : "
          + logFile.getName(), e);
    } else {
      throw new IOException("Exception reading log file. It might be because log "
          + "file was aggregated : " + logFile.getName(), e);
    }
  }
}
 
Example 4
Source File: ContainerPage.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
protected void render(Block html) {
  ContainerId containerID;
  try {
    containerID = ConverterUtils.toContainerId($(CONTAINER_ID));
  } catch (IllegalArgumentException e) {
    html.p()._("Invalid containerId " + $(CONTAINER_ID))._();
    return;
  }

  DIV<Hamlet> div = html.div("#content");
  Container container = this.nmContext.getContainers().get(containerID);
  if (container == null) {
    div.h1("Unknown Container. Container might have completed, "
            + "please go back to the previous page and retry.")._();
    return;
  }
  ContainerInfo info = new ContainerInfo(this.nmContext, container);

  info("Container information")
    ._("ContainerID", info.getId())
    ._("ContainerState", info.getState())
    ._("ExitStatus", info.getExitStatus())
    ._("Diagnostics", info.getDiagnostics())
    ._("User", info.getUser())
    ._("TotalMemoryNeeded", info.getMemoryNeeded())
    ._("TotalVCoresNeeded", info.getVCoresNeeded())
    ._("logs", info.getShortLogLink(), "Link to logs");
  html._(InfoBlock.class);
}
 
Example 5
Source File: TestMRAppMaster.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testMRAppMasterMissingStaging() throws IOException,
    InterruptedException {
  String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
  String containerIdStr = "container_1317529182569_0004_000002_1";
  String userName = "TestAppMasterUser";
  JobConf conf = new JobConf();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  ApplicationAttemptId applicationAttemptId = ConverterUtils
      .toApplicationAttemptId(applicationAttemptIdStr);

  //Delete the staging directory
  File dir = new File(stagingDir);
  if(dir.exists()) {
    FileUtils.deleteDirectory(dir);
  }
  
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  MRAppMaster appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
          System.currentTimeMillis(), false, false);
  boolean caught = false;
  try {
    MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
  } catch (IOException e) {
    //The IO Exception is expected
    LOG.info("Caught expected Exception", e);
    caught = true;
  }
  assertTrue(caught);
  assertTrue(appMaster.errorHappenedShutDown);
  //Copying the history file is disabled, but it is not really visible from 
  //here
  assertEquals(JobStateInternal.ERROR, appMaster.forcedState);
  appMaster.stop();
}
 
Example 6
Source File: TestMRAppMaster.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testMRAppMasterMidLock() throws IOException,
    InterruptedException {
  String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
  String containerIdStr = "container_1317529182569_0004_000002_1";
  String userName = "TestAppMasterUser";
  JobConf conf = new JobConf();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  ApplicationAttemptId applicationAttemptId = ConverterUtils
      .toApplicationAttemptId(applicationAttemptIdStr);
  JobId jobId =  TypeConverter.toYarn(
      TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
  Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
  FileSystem fs = FileSystem.get(conf);
  //Create the file, but no end file so we should unregister with an error.
  fs.create(start).close();
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  MRAppMaster appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
          System.currentTimeMillis(), false, false);
  boolean caught = false;
  try {
    MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
  } catch (IOException e) {
    //The IO Exception is expected
    LOG.info("Caught expected Exception", e);
    caught = true;
  }
  assertTrue(caught);
  assertTrue(appMaster.errorHappenedShutDown);
  assertEquals(JobStateInternal.ERROR, appMaster.forcedState);
  appMaster.stop();

  // verify the final status is FAILED
  verifyFailedStatus((MRAppMasterTest)appMaster, "FAILED");
}
 
Example 7
Source File: TestYarnContainerHeartbeatServlet.java    From samza with Apache License 2.0 5 votes vote down vote up
@Before
public void setup()
    throws Exception {
  container = mock(YarnContainer.class);
  ReadableMetricsRegistry registry = new MetricsRegistryMap("test-registry");

  yarnAppState =
      new YarnAppState(-1, ConverterUtils.toContainerId("container_1350670447861_0003_01_000001"), "testHost", 1, 1);
  webApp = new HttpServer("/", 0, "", new ServletHolder(new DefaultServlet()));
  webApp.addServlet("/", new YarnContainerHeartbeatServlet(yarnAppState, registry));
  webApp.start();
  mapper = new ObjectMapper();
}
 
Example 8
Source File: TestMRAppMaster.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testMRAppMasterMissingStaging() throws IOException,
    InterruptedException {
  String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
  String containerIdStr = "container_1317529182569_0004_000002_1";
  String userName = "TestAppMasterUser";
  JobConf conf = new JobConf();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  ApplicationAttemptId applicationAttemptId = ConverterUtils
      .toApplicationAttemptId(applicationAttemptIdStr);

  //Delete the staging directory
  File dir = new File(stagingDir);
  if(dir.exists()) {
    FileUtils.deleteDirectory(dir);
  }
  
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  MRAppMaster appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
          System.currentTimeMillis(), false, false);
  boolean caught = false;
  try {
    MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
  } catch (IOException e) {
    //The IO Exception is expected
    LOG.info("Caught expected Exception", e);
    caught = true;
  }
  assertTrue(caught);
  assertTrue(appMaster.errorHappenedShutDown);
  //Copying the history file is disabled, but it is not really visible from 
  //here
  assertEquals(JobStateInternal.ERROR, appMaster.forcedState);
  appMaster.stop();
}
 
Example 9
Source File: NMLeveldbStateStoreService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static void loadContainerToken(RecoveredContainerTokensState state,
    String key, String containerIdStr, byte[] value) throws IOException {
  ContainerId containerId;
  Long expTime;
  try {
    containerId = ConverterUtils.toContainerId(containerIdStr);
    expTime = Long.parseLong(asString(value));
  } catch (IllegalArgumentException e) {
    throw new IOException("Bad container token state for " + key, e);
  }
  state.activeTokens.put(containerId, expTime);
}
 
Example 10
Source File: NMController.java    From big-c with Apache License 2.0 5 votes vote down vote up
public void logs() {
  String containerIdStr = $(CONTAINER_ID);
  ContainerId containerId = null;
  try {
    containerId = ConverterUtils.toContainerId(containerIdStr);
  } catch (IllegalArgumentException e) {
    render(ContainerLogsPage.class);
    return;
  }
  ApplicationId appId =
      containerId.getApplicationAttemptId().getApplicationId();
  Application app = nmContext.getApplications().get(appId);
  if (app == null
      && nmConf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
          YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED)) {
    String logServerUrl = nmConf.get(YarnConfiguration.YARN_LOG_SERVER_URL);
    String redirectUrl = null;
    if (logServerUrl == null || logServerUrl.isEmpty()) {
      redirectUrl = "false";
    } else {
      redirectUrl =
          url(logServerUrl, nmContext.getNodeId().toString(), containerIdStr,
              containerIdStr, $(APP_OWNER));
    }
    set(ContainerLogsPage.REDIRECT_URL, redirectUrl);
  }
  render(ContainerLogsPage.class);
}
 
Example 11
Source File: GobblinYarnTaskRunner.java    From incubator-gobblin with Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) {
  Options options = buildOptions();
  try {
    CommandLine cmd = new DefaultParser().parse(options, args);
    if (!cmd.hasOption(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME) || !cmd
        .hasOption(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_OPTION_NAME) || !cmd
  .hasOption(GobblinClusterConfigurationKeys.APPLICATION_ID_OPTION_NAME)) {
      printUsage(options);
      System.exit(1);
    }

    Log4jConfigurationHelper.updateLog4jConfiguration(GobblinTaskRunner.class,
        GobblinYarnConfigurationKeys.GOBBLIN_YARN_LOG4J_CONFIGURATION_FILE,
        GobblinYarnConfigurationKeys.GOBBLIN_YARN_LOG4J_CONFIGURATION_FILE);

    LOGGER.info(JvmUtils.getJvmInputArguments());

    ContainerId containerId =
        ConverterUtils.toContainerId(System.getenv().get(ApplicationConstants.Environment.CONTAINER_ID.key()));
    String applicationName = cmd.getOptionValue(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME);
    String applicationId = cmd.getOptionValue(GobblinClusterConfigurationKeys.APPLICATION_ID_OPTION_NAME);
    String helixInstanceName = cmd.getOptionValue(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_OPTION_NAME);
    String helixInstanceTags = cmd.getOptionValue(GobblinClusterConfigurationKeys.HELIX_INSTANCE_TAGS_OPTION_NAME);

    Config config = ConfigFactory.load();
    if (!Strings.isNullOrEmpty(helixInstanceTags)) {
      config = config.withValue(GobblinClusterConfigurationKeys.HELIX_INSTANCE_TAGS_KEY, ConfigValueFactory.fromAnyRef(helixInstanceTags));
    }

    GobblinTaskRunner gobblinTaskRunner =
        new GobblinYarnTaskRunner(applicationName, applicationId, helixInstanceName, containerId, config,
            Optional.<Path>absent());
    gobblinTaskRunner.start();
  } catch (ParseException pe) {
    printUsage(options);
    System.exit(1);
  } catch (Throwable t) {
    // Ideally, we should not be catching non-recoverable exceptions and errors. However,
    // simply propagating the exception may prevent the container exit due to the presence of non-daemon threads present
    // in the application. Hence, we catch this exception to invoke System.exit() which in turn ensures that all non-daemon threads are killed.
    LOGGER.error("Exception encountered: {}", t);
    System.exit(1);
  }
}
 
Example 12
Source File: TaskAttemptStartedEvent.java    From big-c with Apache License 2.0 4 votes vote down vote up
/** Get the ContainerId */
public ContainerId getContainerId() {
  return ConverterUtils.toContainerId(datum.containerId.toString());
}
 
Example 13
Source File: DAGAppMaster.java    From tez with Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) {
  try {
    Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
    final String pid = System.getenv().get("JVM_PID");
    String containerIdStr =
        System.getenv(Environment.CONTAINER_ID.name());
    String nodeHostString = System.getenv(Environment.NM_HOST.name());
    String nodePortString = System.getenv(Environment.NM_PORT.name());
    String nodeHttpPortString =
        System.getenv(Environment.NM_HTTP_PORT.name());
    String appSubmitTimeStr =
        System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV);
    String clientVersion = System.getenv(TezConstants.TEZ_CLIENT_VERSION_ENV);
    if (clientVersion == null) {
      clientVersion = VersionInfo.UNKNOWN;
    }

    validateInputParam(appSubmitTimeStr,
        ApplicationConstants.APP_SUBMIT_TIME_ENV);

    ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
    ApplicationAttemptId applicationAttemptId =
        containerId.getApplicationAttemptId();

    long appSubmitTime = Long.parseLong(appSubmitTimeStr);

    String jobUserName = System
        .getenv(ApplicationConstants.Environment.USER.name());

    // Command line options
    Options opts = new Options();
    opts.addOption(TezConstants.TEZ_SESSION_MODE_CLI_OPTION,
        false, "Run Tez Application Master in Session mode");

    CommandLine cliParser = new GnuParser().parse(opts, args);
    boolean sessionModeCliOption = cliParser.hasOption(TezConstants.TEZ_SESSION_MODE_CLI_OPTION);

    LOG.info("Creating DAGAppMaster for "
        + "applicationId=" + applicationAttemptId.getApplicationId()
        + ", attemptNum=" + applicationAttemptId.getAttemptId()
        + ", AMContainerId=" + containerId
        + ", jvmPid=" + pid
        + ", userFromEnv=" + jobUserName
        + ", cliSessionOption=" + sessionModeCliOption
        + ", pwd=" + System.getenv(Environment.PWD.name())
        + ", localDirs=" + System.getenv(Environment.LOCAL_DIRS.name())
        + ", logDirs=" + System.getenv(Environment.LOG_DIRS.name()));

    // TODO Does this really need to be a YarnConfiguration ?
    Configuration conf = new Configuration(new YarnConfiguration());

    ConfigurationProto confProto =
        TezUtilsInternal.readUserSpecifiedTezConfiguration(System.getenv(Environment.PWD.name()));
    TezUtilsInternal.addUserSpecifiedTezConfiguration(conf, confProto.getConfKeyValuesList());

    AMPluginDescriptorProto amPluginDescriptorProto = null;
    if (confProto.hasAmPluginDescriptor()) {
      amPluginDescriptorProto = confProto.getAmPluginDescriptor();
    }

    UserGroupInformation.setConfiguration(conf);
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();

    TezUtilsInternal.setSecurityUtilConfigration(LOG, conf);

    DAGAppMaster appMaster =
        new DAGAppMaster(applicationAttemptId, containerId, nodeHostString,
            Integer.parseInt(nodePortString),
            Integer.parseInt(nodeHttpPortString), new SystemClock(), appSubmitTime,
            sessionModeCliOption,
            System.getenv(Environment.PWD.name()),
            TezCommonUtils.getTrimmedStrings(System.getenv(Environment.LOCAL_DIRS.name())),
            TezCommonUtils.getTrimmedStrings(System.getenv(Environment.LOG_DIRS.name())),
            clientVersion, credentials, jobUserName, amPluginDescriptorProto);
    ShutdownHookManager.get().addShutdownHook(
      new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);

    // log the system properties
    if (LOG.isInfoEnabled()) {
      String systemPropsToLog = TezCommonUtils.getSystemPropertiesToLog(conf);
      if (systemPropsToLog != null) {
        LOG.info(systemPropsToLog);
      }
    }

    initAndStartAppMaster(appMaster, conf);

  } catch (Throwable t) {
    LOG.error("Error starting DAGAppMaster", t);
    System.exit(1);
  }
}
 
Example 14
Source File: AMStartedEvent.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * @return the ContainerId for the MRAppMaster.
 */
public ContainerId getContainerId() {
  return ConverterUtils.toContainerId(datum.containerId.toString());
}
 
Example 15
Source File: ContainerLaunchedEvent.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
public void fromProto(ContainerLaunchedProto proto) {
  this.containerId = ConverterUtils.toContainerId(proto.getContainerId());
  launchTime = proto.getLaunchTime();
  this.applicationAttemptId = ConverterUtils.toApplicationAttemptId(
      proto.getApplicationAttemptId());
}
 
Example 16
Source File: XLearningContainerId.java    From XLearning with Apache License 2.0 4 votes vote down vote up
@Override
public void readFields(DataInput dataInput) throws IOException {
  this.containerId = ConverterUtils.toContainerId(Text.readString(dataInput));
}
 
Example 17
Source File: YarnClusterResourceManager.java    From samza with Apache License 2.0 4 votes vote down vote up
/**
 * Creates an YarnClusterResourceManager from config, a jobModelReader and a callback.
 * @param config to instantiate the cluster manager with
 * @param jobModelManager the jobModel manager to get the job model (mostly for the UI)
 * @param callback the callback to receive events from Yarn.
 * @param samzaAppState samza app state for display in the UI
 */
public YarnClusterResourceManager(Config config, JobModelManager jobModelManager,
    ClusterResourceManager.Callback callback, SamzaApplicationState samzaAppState) {
  super(callback);
  yarnConfiguration = new YarnConfiguration();
  yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());

  // Use the Samza job config "fs.<scheme>.impl" and "fs.<scheme>.impl.*" for YarnConfiguration
  FileSystemImplConfig fsImplConfig = new FileSystemImplConfig(config);
  fsImplConfig.getSchemes().forEach(
    scheme -> {
      fsImplConfig.getSchemeConfig(scheme).forEach(
        (confKey, confValue) -> yarnConfiguration.set(confKey, confValue)
      );
    }
  );

  MetricsRegistryMap registry = new MetricsRegistryMap();
  metrics = new SamzaAppMasterMetrics(config, samzaAppState, registry);

  // parse configs from the Yarn environment
  String containerIdStr = System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString());
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  String nodeHostString = System.getenv(ApplicationConstants.Environment.NM_HOST.toString());
  String nodePortString = System.getenv(ApplicationConstants.Environment.NM_PORT.toString());
  String nodeHttpPortString = System.getenv(ApplicationConstants.Environment.NM_HTTP_PORT.toString());

  int nodePort = Integer.parseInt(nodePortString);
  int nodeHttpPort = Integer.parseInt(nodeHttpPortString);
  YarnConfig yarnConfig = new YarnConfig(config);
  this.yarnConfig = yarnConfig;
  this.config = config;
  int interval = yarnConfig.getAMPollIntervalMs();

  //Instantiate the AM Client.
  this.amClient = AMRMClientAsync.createAMRMClientAsync(interval, this);

  this.state = new YarnAppState(-1, containerId, nodeHostString, nodePort, nodeHttpPort);

  log.info("Initialized YarnAppState: {}", state.toString());
  this.service = new SamzaYarnAppMasterService(config, samzaAppState, this.state, registry, yarnConfiguration);

  log.info("Container ID: {}, Nodehost:  {} , Nodeport : {} , NodeHttpport: {}", containerIdStr, nodeHostString, nodePort, nodeHttpPort);
  ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config);
  this.lifecycle = new SamzaYarnAppMasterLifecycle(
      clusterManagerConfig.getContainerMemoryMb(),
      clusterManagerConfig.getNumCores(),
      samzaAppState,
      state,
      amClient
  );
  this.nmClientAsync = NMClientAsync.createNMClientAsync(this);

}
 
Example 18
Source File: StreamingAppMaster.java    From attic-apex-core with Apache License 2.0 4 votes vote down vote up
/**
 * @param args
 *          Command line args
 * @throws Throwable
 */
public static void main(final String[] args) throws Throwable
{
  LoggerUtil.setupMDC("master");
  StdOutErrLog.tieSystemOutAndErrToLog();
  LOG.info("Master starting with classpath: {}", System.getProperty("java.class.path"));

  LOG.info("version: {}", VersionInfo.APEX_VERSION.getBuildVersion());
  StringWriter sw = new StringWriter();
  for (Map.Entry<String, String> e : System.getenv().entrySet()) {
    sw.append("\n").append(e.getKey()).append("=").append(e.getValue());
  }
  LOG.info("appmaster env:" + sw.toString());

  Options opts = new Options();
  opts.addOption("app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes");

  opts.addOption("help", false, "Print usage");
  CommandLine cliParser = new GnuParser().parse(opts, args);

  // option "help" overrides and cancels any run
  if (cliParser.hasOption("help")) {
    new HelpFormatter().printHelp("ApplicationMaster", opts);
    return;
  }

  Map<String, String> envs = System.getenv();
  ApplicationAttemptId appAttemptID = Records.newRecord(ApplicationAttemptId.class);
  if (!envs.containsKey(Environment.CONTAINER_ID.name())) {
    if (cliParser.hasOption("app_attempt_id")) {
      String appIdStr = cliParser.getOptionValue("app_attempt_id", "");
      appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
    } else {
      throw new IllegalArgumentException("Application Attempt Id not set in the environment");
    }
  } else {
    ContainerId containerId = ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name()));
    appAttemptID = containerId.getApplicationAttemptId();
  }

  boolean result = false;
  StreamingAppMasterService appMaster = null;
  try {
    appMaster = new StreamingAppMasterService(appAttemptID);
    LOG.info("Initializing Application Master.");

    Configuration conf = new YarnConfiguration();
    appMaster.init(conf);
    appMaster.start();
    result = appMaster.run();
  } catch (Throwable t) {
    LOG.error("Exiting Application Master", t);
    System.exit(1);
  } finally {
    if (appMaster != null) {
      appMaster.stop();
    }
  }

  if (result) {
    LOG.info("Application Master completed.");
    System.exit(0);
  } else {
    LOG.info("Application Master failed.");
    System.exit(2);
  }
}
 
Example 19
Source File: AMStartedEvent.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * @return the ContainerId for the MRAppMaster.
 */
public ContainerId getContainerId() {
  return ConverterUtils.toContainerId(datum.containerId.toString());
}
 
Example 20
Source File: TaskAttemptListenerImpTezDag.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
@Override
public ContainerTask getTask(ContainerContext containerContext)
    throws IOException {

  ContainerTask task = null;

  if (containerContext == null || containerContext.getContainerIdentifier() == null) {
    LOG.info("Invalid task request with an empty containerContext or containerId");
    task = TASK_FOR_INVALID_JVM;
  } else {
    ContainerId containerId = ConverterUtils.toContainerId(containerContext
        .getContainerIdentifier());
    if (LOG.isDebugEnabled()) {
      LOG.debug("Container with id: " + containerId + " asked for a task");
    }
    if (!registeredContainers.containsKey(containerId)) {
      if(context.getAllContainers().get(containerId) == null) {
        LOG.info("Container with id: " + containerId
            + " is invalid and will be killed");
      } else {
        LOG.info("Container with id: " + containerId
            + " is valid, but no longer registered, and will be killed");
      }
      task = TASK_FOR_INVALID_JVM;
    } else {
      pingContainerHeartbeatHandler(containerId);
      AMContainerTask taskContext = pullTaskAttemptContext(containerId);
      if (taskContext.shouldDie()) {
        LOG.info("No more tasks for container with id : " + containerId
            + ". Asking it to die");
        task = TASK_FOR_INVALID_JVM; // i.e. ask the child to die.
      } else {
        if (taskContext.getTask() == null) {
          if (LOG.isDebugEnabled()) {
            LOG.debug("No task currently assigned to Container with id: "
                + containerId);
          }
        } else {
          registerTaskAttempt(taskContext.getTask().getTaskAttemptID(),
              containerId);
          task = new ContainerTask(taskContext.getTask(), false,
              convertLocalResourceMap(taskContext.getAdditionalResources()),
              taskContext.getCredentials(), taskContext.haveCredentialsChanged());
          context.getEventHandler().handle(
              new TaskAttemptEventStartedRemotely(taskContext.getTask()
                  .getTaskAttemptID(), containerId, context
                  .getApplicationACLs()));
          LOG.info("Container with id: " + containerId + " given task: "
              + taskContext.getTask().getTaskAttemptID());
        }
      }
    }
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("getTask returning task: " + task);
  }
  return task;
}