Java Code Examples for org.apache.hadoop.yarn.conf.YarnConfiguration#addResource()

The following examples show how to use org.apache.hadoop.yarn.conf.YarnConfiguration#addResource() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: AbstractYarnClusterDescriptor.java From Flink-CEPplus with Apache License 2.0

6 votes

public AbstractYarnClusterDescriptor(
		Configuration flinkConfiguration,
		YarnConfiguration yarnConfiguration,
		String configurationDirectory,
		YarnClient yarnClient,
		boolean sharedYarnClient) {

	this.yarnConfiguration = Preconditions.checkNotNull(yarnConfiguration);

	// for unit tests only
	if (System.getenv("IN_TESTS") != null) {
		try {
			yarnConfiguration.addResource(new File(System.getenv("YARN_CONF_DIR"), "yarn-site.xml").toURI().toURL());
		} catch (Throwable t) {
			throw new RuntimeException("Error", t);
		}
	}

	this.yarnClient = Preconditions.checkNotNull(yarnClient);
	this.sharedYarnClient = sharedYarnClient;

	this.flinkConfiguration = Preconditions.checkNotNull(flinkConfiguration);
	userJarInclusion = getUserJarInclusionMode(flinkConfiguration);

	this.configurationDirectory = Preconditions.checkNotNull(configurationDirectory);
}

Example 2

Source File: JstormMaster.java From PoseidonX with Apache License 2.0

6 votes

/**
 * JstormAM 的构造函数
 */
public JstormMaster() throws IOException {
    conf = new YarnConfiguration();
    Path jstormYarnConfPath = new Path("jstorm-yarn.xml");
    conf.addResource(jstormYarnConfPath);

    JstormAMContext.serviceUserName = RegistryUtils.currentUser();
    LOG.error("### JstormAMContext.serviceUserName:["+JstormAMContext.serviceUserName+"]");
    registryClient = RegistryClient.init(JstormAMConstant.REGISTRY_JSTORM_YARN,JstormAMContext.serviceUserName,conf);
    slotPortsView = new SlotPortsView(registryClient);
    slotPortsView.setMinPort(JstormAMConstant.SUPERVISOR_MIN_PORT);
    slotPortsView.setMaxPort(JstormAMConstant.SUPERVISOR_MAX_PORT);
    JstormAMContext.requestBlockingQueue = new LinkedBlockingQueue<ContainerRequest>();

    JstormAMContext.currentNimbusNum = new AtomicInteger(0);
    JstormAMContext.currentSupervisorNum = new AtomicInteger(0);
}

Example 3

Source File: TestTimelineWebServicesWithSSL.java From hadoop with Apache License 2.0

6 votes

@BeforeClass
public static void setupServer() throws Exception {
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
  conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
      MemoryTimelineStore.class, TimelineStore.class);
  conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, "HTTPS_ONLY");

  File base = new File(BASEDIR);
  FileUtil.fullyDelete(base);
  base.mkdirs();
  keystoresDir = new File(BASEDIR).getAbsolutePath();
  sslConfDir =
      KeyStoreTestUtil.getClasspathDir(TestTimelineWebServicesWithSSL.class);

  KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false);
  conf.addResource("ssl-server.xml");
  conf.addResource("ssl-client.xml");

  timelineServer = new ApplicationHistoryServer();
  timelineServer.init(conf);
  timelineServer.start();
  store = timelineServer.getTimelineStore();
}

Example 4

Source File: TestTimelineWebServicesWithSSL.java From big-c with Apache License 2.0

6 votes

@BeforeClass
public static void setupServer() throws Exception {
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
  conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
      MemoryTimelineStore.class, TimelineStore.class);
  conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, "HTTPS_ONLY");

  File base = new File(BASEDIR);
  FileUtil.fullyDelete(base);
  base.mkdirs();
  keystoresDir = new File(BASEDIR).getAbsolutePath();
  sslConfDir =
      KeyStoreTestUtil.getClasspathDir(TestTimelineWebServicesWithSSL.class);

  KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false);
  conf.addResource("ssl-server.xml");
  conf.addResource("ssl-client.xml");

  timelineServer = new ApplicationHistoryServer();
  timelineServer.init(conf);
  timelineServer.start();
  store = timelineServer.getTimelineStore();
}

Example 5

Source File: Executor.java From jstorm with Apache License 2.0

6 votes

public Executor(String instancName, String shellCommand, STARTType startType, String runningContainer,
                String localDir, String deployPath,
                String hadoopHome, String javaHome, String pythonHome, String dstPath, String portList, String shellArgs,
                String ExecShellStringPath, String applicationId, String supervisorLogviewPort, String nimbusThriftPort) {
    executorMeta = new ExecutorMeta(instancName, shellCommand, startType, runningContainer,
            localDir, deployPath, hadoopHome, javaHome, pythonHome, dstPath, portList, shellArgs,
            ExecShellStringPath, applicationId, supervisorLogviewPort, nimbusThriftPort);

    conf = new YarnConfiguration();
    Path yarnSite = new Path(hadoopHome + JOYConstants.YARN_SITE_PATH);
    conf.addResource(yarnSite);

    //get first log dir
    logDir = conf.get(JOYConstants.YARN_NM_LOG, JOYConstants.YARN_NM_LOG_DIR).split(JOYConstants.COMMA)[0] + JOYConstants.BACKLASH + applicationId + JOYConstants.BACKLASH + runningContainer;
    //Setup RegistryOperations
    registryOperations = RegistryOperationsFactory.createInstance(JOYConstants.YARN_REGISTRY, conf);
    try {
        setupInitialRegistryPaths();
    } catch (IOException e) {
        e.printStackTrace();
    }
    registryOperations.start();
}

Example 6

Source File: AthenaXConfiguration.java From AthenaX with Apache License 2.0

5 votes

public YarnClusterConfiguration toYarnClusterConfiguration() {
  Preconditions.checkNotNull(yarnSite, "yarn.site.location is not configured");
  Preconditions.checkNotNull(homeDir, "athenax.home.dir is not configured");
  Preconditions.checkNotNull(flinkUberJar, "flink.uber.jar.location is not configured");

  YarnConfiguration yarnConf = new YarnConfiguration();
  yarnConf.addResource(new Path(URI.create(yarnSite)));

  return new YarnClusterConfiguration(
      yarnConf, homeDir, new Path(flinkUberJar),
      resourcesToLocalize.stream().map(x -> new Path(URI.create(x))).collect(Collectors.toSet()),
      additionalJars.stream().map(x -> new Path(URI.create(x))).collect(Collectors.toSet()));
}

Example 7

Source File: AzkabanGobblinLocalYarnAppLauncher.java From incubator-gobblin with Apache License 2.0

5 votes

@Override
protected YarnConfiguration initYarnConf(Properties gobblinProps) {
  YarnConfiguration yarnConfiguration = super.initYarnConf(gobblinProps);
  if (gobblinProps.containsKey("yarn-site-address")) {
    yarnConfiguration.addResource(new Path(gobblinProps.getProperty("yarn-site-address")));
  } else {
    yarnConfiguration.set("yarn.resourcemanager.connect.max-wait.ms", "10000");
    yarnConfiguration.set("yarn.nodemanager.resource.memory-mb", "512");
    yarnConfiguration.set("yarn.scheduler.maximum-allocation-mb", "1024");
  }
  return yarnConfiguration;
}

Example 8

Source File: JstormMaster.java From jstorm with Apache License 2.0

4 votes

public JstormMaster() {
    // Set up the configuration
    conf = new YarnConfiguration();
    Path jstormyarnConfPath = new Path("jstorm-yarn.xml");
    conf.addResource(jstormyarnConfPath);
}