org.apache.hadoop.yarn.conf.YarnConfiguration Java Examples

The following examples show how to use org.apache.hadoop.yarn.conf.YarnConfiguration. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestLocalDirsHandlerService.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testValidPathsDirHandlerService() throws Exception {
  Configuration conf = new YarnConfiguration();
  String localDir1 = new File("file:///" + testDir, "localDir1").getPath();
  String localDir2 = new File("hdfs:///" + testDir, "localDir2").getPath();
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir1 + "," + localDir2);
  String logDir1 = new File("file:///" + testDir, "logDir1").getPath();
  conf.set(YarnConfiguration.NM_LOG_DIRS, logDir1);
  LocalDirsHandlerService dirSvc = new LocalDirsHandlerService();
  try {
    dirSvc.init(conf);
    Assert.fail("Service should have thrown an exception due to wrong URI");
  } catch (YarnRuntimeException e) {
  }
  Assert.assertEquals("Service should not be inited",
                      STATE.STOPPED,
                      dirSvc.getServiceState());
  dirSvc.close();
}
 
Example #2
Source File: AdminService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings(
    RefreshUserToGroupsMappingsRequest request)
    throws YarnException, IOException {
  String argName = "refreshUserToGroupsMappings";
  UserGroupInformation user = checkAcls(argName);

  checkRMStatus(user.getShortUserName(), argName, "refresh user-groups.");

  Groups.getUserToGroupsMappingService(
      getConfiguration(new Configuration(false),
          YarnConfiguration.CORE_SITE_CONFIGURATION_FILE)).refresh();

  RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService");

  return recordFactory.newRecordInstance(
      RefreshUserToGroupsMappingsResponse.class);
}
 
Example #3
Source File: TestSharedCacheUploader.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * If resource is public, verifyAccess should succeed
 */
@Test
public void testVerifyAccessPublicResource() throws Exception {
  Configuration conf = new Configuration();
  conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true);
  LocalResource resource = mock(LocalResource.class);
  // give public visibility
  when(resource.getVisibility()).thenReturn(LocalResourceVisibility.PUBLIC);
  Path localPath = mock(Path.class);
  when(localPath.getName()).thenReturn("foo.jar");
  String user = "joe";
  SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class);
  FileSystem fs = mock(FileSystem.class);
  FileSystem localFs = FileSystem.getLocal(conf);
  SharedCacheUploader spied =
      createSpiedUploader(resource, localPath, user, conf, scmClient, fs,
          localFs);

  assertTrue(spied.verifyAccess());
}
 
Example #4
Source File: TestResourceManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testNMExpiryAndHeartbeatIntervalsValidation() throws Exception {
  Configuration conf = new YarnConfiguration();
  conf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 1000);
  conf.setLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 1001);
  resourceManager = new ResourceManager();;
  try {
    resourceManager.init(conf);
  } catch (YarnRuntimeException e) {
    // Exception is expected.
    if (!e.getMessage().startsWith("Nodemanager expiry interval should be no"
        + " less than heartbeat interval")) {
      throw e;
    }
  }
}
 
Example #5
Source File: TestRMWebServices.java    From big-c with Apache License 2.0 6 votes vote down vote up
public void verifyClusterSchedulerFifoGeneric(String type, String state,
    float capacity, float usedCapacity, int minQueueCapacity,
    int maxQueueCapacity, int numNodes, int usedNodeCapacity,
    int availNodeCapacity, int totalNodeCapacity, int numContainers)
    throws JSONException, Exception {

  assertEquals("type doesn't match", "fifoScheduler", type);
  assertEquals("qstate doesn't match", QueueState.RUNNING.toString(), state);
  assertEquals("capacity doesn't match", 1.0, capacity, 0.0);
  assertEquals("usedCapacity doesn't match", 0.0, usedCapacity, 0.0);
  assertEquals(
      "minQueueMemoryCapacity doesn't match",
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
      minQueueCapacity);
  assertEquals("maxQueueMemoryCapacity doesn't match",
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
      maxQueueCapacity);
  assertEquals("numNodes doesn't match", 0, numNodes);
  assertEquals("usedNodeCapacity doesn't match", 0, usedNodeCapacity);
  assertEquals("availNodeCapacity doesn't match", 0, availNodeCapacity);
  assertEquals("totalNodeCapacity doesn't match", 0, totalNodeCapacity);
  assertEquals("numContainers doesn't match", 0, numContainers);

}
 
Example #6
Source File: TestFairScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test  
public void testMinZeroResourcesSettings() throws IOException {  
  scheduler = new FairScheduler();
  YarnConfiguration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 0);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_GCORES, 0);
  conf.setInt(
    FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512);
  conf.setInt(
    FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 2);
  conf.setInt(
    FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_GCORES, 1);
  scheduler.init(conf);
  scheduler.reinitialize(conf, null);
  Assert.assertEquals(0, scheduler.getMinimumResourceCapability().getMemory());
  Assert.assertEquals(0, scheduler.getMinimumResourceCapability().getVirtualCores());
  Assert.assertEquals(0, scheduler.getMinimumResourceCapability().getGpuCores());
  Assert.assertEquals(512, scheduler.getIncrementResourceCapability().getMemory());
  Assert.assertEquals(2, scheduler.getIncrementResourceCapability().getVirtualCores());
  Assert.assertEquals(1, scheduler.getIncrementResourceCapability().getGpuCores());
}
 
Example #7
Source File: TestFifoScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void testMinimumAllocation(YarnConfiguration conf, int testAlloc)
    throws Exception {
  MockRM rm = new MockRM(conf);
  rm.start();

  // Register node1
  MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);

  // Submit an application
  RMApp app1 = rm.submitApp(testAlloc);

  // kick the scheduling
  nm1.nodeHeartbeat(true);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
  am1.registerAppAttempt();
  SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
      nm1.getNodeId());

  int checkAlloc =
      conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
  Assert.assertEquals(checkAlloc, report_nm1.getUsedResource().getMemory());

  rm.stop();
}
 
Example #8
Source File: TestSharedCacheUploader.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * If the localPath does not exists, getActualPath should get to one level
 * down
 */
@Test
public void testGetActualPath() throws Exception {
  Configuration conf = new Configuration();
  conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true);
  LocalResource resource = mock(LocalResource.class);
  // give public visibility
  when(resource.getVisibility()).thenReturn(LocalResourceVisibility.PUBLIC);
  Path localPath = new Path("foo.jar");
  String user = "joe";
  SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class);
  FileSystem fs = mock(FileSystem.class);
  FileSystem localFs = mock(FileSystem.class);
  // stub it to return a status that indicates a directory
  FileStatus status = mock(FileStatus.class);
  when(status.isDirectory()).thenReturn(true);
  when(localFs.getFileStatus(localPath)).thenReturn(status);
  SharedCacheUploader spied =
      createSpiedUploader(resource, localPath, user, conf, scmClient, fs,
          localFs);

  Path actualPath = spied.getActualPath();
  assertEquals(actualPath.getName(), localPath.getName());
  assertEquals(actualPath.getParent().getName(), localPath.getName());
}
 
Example #9
Source File: TestYarnService.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
@Test
public void testDistroDefaultsOverwrite() throws Exception {
  assumeNonMaprProfile();

  YarnController controller = Mockito.mock(YarnController.class);
  YarnService yarnService = new YarnService(new TestListener(), controller, Mockito.mock(NodeProvider.class));

  Cluster myCluster = createCluster();
  List<Property> props = myCluster.getClusterConfig().getSubPropertyList();
  props.add(new Property(YarnDefaultsConfigurator.SPILL_PATH, "/abc/bcd"));
  props.add(new Property(YarnDefaultsConfigurator.JAVA_LOGIN, "/abc/bcd/login.conf"));
  myCluster.getClusterConfig().setDistroType(DistroType.HDP).setIsSecure(true);
  YarnConfiguration myYarnConfig = new YarnConfiguration();
  yarnService.updateYarnConfiguration(myCluster, myYarnConfig);

  assertEquals("/abc/bcd/login.conf", myYarnConfig.get(YarnDefaultsConfigurator.JAVA_LOGIN));
  assertNull(myYarnConfig.get(YarnDefaultsConfigurator.ZK_SASL_CLIENT));
  assertNull(myYarnConfig.get(YarnDefaultsConfigurator.ZK_SASL_CLIENT_CONFIG));
  assertNull(myYarnConfig.get(YarnDefaultsConfigurator.ZK_SASL_PROVIDER));
  assertEquals("/abc/bcd", myYarnConfig.get(YarnDefaultsConfigurator.SPILL_PATH));
}
 
Example #10
Source File: TestWorkPreservingRMRestart.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Test validateAndCreateResourceRequest fails on recovery, app should ignore
 * this Exception and continue
 */
@Test (timeout = 30000)
public void testAppFailToValidateResourceRequestOnRecovery() throws Exception{
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1 = rm1.submitApp(200);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);

  // Change the config so that validateAndCreateResourceRequest throws
  // exception on recovery
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 50);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 100);

  rm2 = new MockRM(conf, memStore);
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  rm2.start();
}
 
Example #11
Source File: FileSystemRMStateStore.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
protected synchronized void startInternal() throws Exception {
  // create filesystem only now, as part of service-start. By this time, RM is
  // authenticated with kerberos so we are good to create a file-system
  // handle.
  Configuration conf = new Configuration(getConfig());
  conf.setBoolean("dfs.client.retry.policy.enabled", true);
  String retryPolicy =
      conf.get(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC,
        YarnConfiguration.DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC);
  conf.set("dfs.client.retry.policy.spec", retryPolicy);

  fs = fsWorkingPath.getFileSystem(conf);
  mkdirsWithRetries(rmDTSecretManagerRoot);
  mkdirsWithRetries(rmAppRoot);
  mkdirsWithRetries(amrmTokenSecretManagerRoot);
}
 
Example #12
Source File: TestRMFailover.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testExplicitFailover()
    throws YarnException, InterruptedException, IOException {
  conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
  cluster.init(conf);
  cluster.start();
  getAdminService(0).transitionToActive(req);
  assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex());
  verifyConnections();

  explicitFailover();
  verifyConnections();

  explicitFailover();
  verifyConnections();
}
 
Example #13
Source File: TestSharedCacheUploaderService.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void startUp() {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.SCM_STORE_CLASS,
      InMemorySCMStore.class.getName());
  conf.set(YarnConfiguration.SHARED_CACHE_ROOT, testDir.getPath());
  AppChecker appChecker = spy(new DummyAppChecker());
  store = new InMemorySCMStore(appChecker);
  store.init(conf);
  store.start();

  service = new SharedCacheUploaderService(store);
  service.init(conf);
  service.start();

  YarnRPC rpc = YarnRPC.create(new Configuration());

  InetSocketAddress scmAddress =
      conf.getSocketAddr(YarnConfiguration.SCM_UPLOADER_SERVER_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_PORT);

  proxy =
      (SCMUploaderProtocol) rpc.getProxy(
          SCMUploaderProtocol.class, scmAddress, conf);
}
 
Example #14
Source File: TestTimelineACLsManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testYarnACLsEnabledForDomain() throws Exception {
  Configuration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
  TimelineACLsManager timelineACLsManager =
      new TimelineACLsManager(conf);
  TimelineDomain domain = new TimelineDomain();
  domain.setOwner("owner");
  Assert.assertTrue(
      "Owner should be allowed to access",
      timelineACLsManager.checkAccess(
          UserGroupInformation.createRemoteUser("owner"), domain));
  Assert.assertFalse(
      "Other shouldn't be allowed to access",
      timelineACLsManager.checkAccess(
          UserGroupInformation.createRemoteUser("other"), domain));
  Assert.assertTrue(
      "Admin should be allowed to access",
      timelineACLsManager.checkAccess(
          UserGroupInformation.createRemoteUser("admin"), domain));
}
 
Example #15
Source File: YarnTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
public static void populateYarnSecureConfigurations(Configuration conf, String principal, String keytab) {

		conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
		conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, "true");

		conf.set(YarnConfiguration.RM_KEYTAB, keytab);
		conf.set(YarnConfiguration.RM_PRINCIPAL, principal);
		conf.set(YarnConfiguration.NM_KEYTAB, keytab);
		conf.set(YarnConfiguration.NM_PRINCIPAL, principal);

		conf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY, principal);
		conf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY, keytab);
		conf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY, principal);
		conf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY, keytab);

		conf.set("hadoop.security.auth_to_local", "RULE:[1:$1] RULE:[2:$1]");
	}
 
Example #16
Source File: ConfiguredRMFailoverProxyProvider.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void init(Configuration configuration, RMProxy<T> rmProxy,
                  Class<T> protocol) {
  this.rmProxy = rmProxy;
  this.protocol = protocol;
  this.rmProxy.checkAllowedProtocols(this.protocol);
  this.conf = new YarnConfiguration(configuration);
  Collection<String> rmIds = HAUtil.getRMHAIds(conf);
  this.rmServiceIds = rmIds.toArray(new String[rmIds.size()]);
  conf.set(YarnConfiguration.RM_HA_ID, rmServiceIds[currentProxyIndex]);

  conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,
      conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES,
          YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES));

  conf.setInt(CommonConfigurationKeysPublic.
      IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
      conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS,
          YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS));
}
 
Example #17
Source File: Client.java    From Scribengin with GNU Affero General Public License v3.0 6 votes vote down vote up
private void setupAppMasterEnv(Map<String, String> appMasterEnv) {
  StringBuilder classPathEnv = new StringBuilder();
  classPathEnv.append(Environment.CLASSPATH.$()).append(File.pathSeparatorChar);
  classPathEnv.append("./*");

  for (String c : conf.getStrings(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
    classPathEnv.append(File.pathSeparatorChar);
    classPathEnv.append(c.trim());
  }

  String envStr = classPathEnv.toString();
  LOG.info("env: " + envStr);
  appMasterEnv.put(Environment.CLASSPATH.name(), envStr);
}
 
Example #18
Source File: TestResourceTrackerService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testNodeRegistrationVersionLessThanRM() throws Exception {
  writeToHostsFile("host2");
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());
  conf.set(YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION,"EqualToRM" );
  rm = new MockRM(conf);
  rm.start();
  String nmVersion = "1.9.9";

  ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = NodeId.newInstance("host2", 1234);
  Resource capability = BuilderUtils.newResource(1024, 1, 1);
  req.setResource(capability);
  req.setNodeId(nodeId);
  req.setHttpPort(1234);
  req.setNMVersion(nmVersion);
  // trying to register a invalid node.
  RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response.getNodeAction());
  Assert.assertTrue("Diagnostic message did not contain: 'Disallowed NodeManager " +
      "Version "+ nmVersion + ", is less than the minimum version'",
      response.getDiagnosticsMessage().contains("Disallowed NodeManager Version " +
          nmVersion + ", is less than the minimum version "));

}
 
Example #19
Source File: NodeManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws IOException {
  Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
  StringUtils.startupShutdownMessage(NodeManager.class, args, LOG);
  NodeManager nodeManager = new NodeManager();
  Configuration conf = new YarnConfiguration();
  new GenericOptionsParser(conf, args);
  nodeManager.initAndStartNodeManager(conf, false);
}
 
Example #20
Source File: TestDockerContainerExecutorWithMocks.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(expected = IllegalArgumentException.class)
public void testContainerLaunchInvalidImage() throws IOException {
  String appSubmitter = "nobody";
  String appId = "APP_ID";
  String containerId = "CONTAINER_ID";
  String testImage = "testrepo.com/test-image rm -rf $HADOOP_PREFIX/*";

  Container container = mock(Container.class, RETURNS_DEEP_STUBS);
  ContainerId cId = mock(ContainerId.class, RETURNS_DEEP_STUBS);
  ContainerLaunchContext context = mock(ContainerLaunchContext.class);
  HashMap<String, String> env = new HashMap<String,String>();

  when(container.getContainerId()).thenReturn(cId);
  when(container.getLaunchContext()).thenReturn(context);
  when(cId.getApplicationAttemptId().getApplicationId().toString()).thenReturn(appId);
  when(cId.toString()).thenReturn(containerId);

  when(context.getEnvironment()).thenReturn(env);
  env.put(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage);
  dockerContainerExecutor.getConf()
    .set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage);
  Path scriptPath = new Path("file:///bin/echo");
  Path tokensPath = new Path("file:///dev/null");

  Path pidFile = new Path(workDir, "pid.txt");

  dockerContainerExecutor.activateContainer(cId, pidFile);
  dockerContainerExecutor.launchContainer(container, scriptPath, tokensPath,
    appSubmitter, appId, workDir, dirsHandler.getLocalDirs(),
    dirsHandler.getLogDirs());
}
 
Example #21
Source File: TezClient.java    From tez with Apache License 2.0 5 votes vote down vote up
/**
 * Alternative to start() that explicitly sets sessionAppId and doesn't start a new AM.
 * The caller of getClient is responsible for initializing the new TezClient with a
 * Configuration compatible with the existing AM. It is expected the caller has cached the
 * original Configuration (e.g. in Zookeeper).
 *
 * In contrast to "start", no resources are localized. It is the responsibility of the caller to
 * ensure that existing localized resources and staging dirs are still valid.
 *
 * @param appId
 * @return 'this' just as a convenience for fluent style chaining
 */
public synchronized TezClient getClient(ApplicationId appId) throws TezException, IOException {
  sessionAppId = appId;
  amConfig.setYarnConfiguration(new YarnConfiguration(amConfig.getTezConfiguration()));
  startFrameworkClient();
  setupJavaOptsChecker();

  if (!isSession) {
    String msg = "Must be in session mode to bind TezClient to existing AM";
    LOG.error(msg);
    throw new IllegalStateException(msg);
  }

  LOG.info("Session mode. Reconnecting to session: " + sessionAppId.toString());

  clientTimeout = amConfig.getTezConfiguration().getInt(
          TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS,
          TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS_DEFAULT);

  try {
    setupApplicationContext();
    ApplicationReport appReport = frameworkClient.getApplicationReport(sessionAppId);
    LOG.info("The url to track the Tez Session: " + appReport.getTrackingUrl());
    sessionStarted.set(true);
  } catch (YarnException e) {
    throw new TezException(e);
  }

  startClientHeartbeat();
  this.stagingFs = FileSystem.get(amConfig.getTezConfiguration());
  return this;
}
 
Example #22
Source File: TestMRHelpers.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() throws IOException {
  try {
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
    dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2)
      .format(true).racks(null).build();
    remoteFs = dfsCluster.getFileSystem();
  } catch (IOException io) {
    throw new RuntimeException("problem starting mini dfs cluster", io);
  }

  Configuration testConf = new YarnConfiguration(
      dfsCluster.getFileSystem().getConf());

  File testConfFile = new File(TEST_ROOT_DIR, "test.xml");
  try {
    testConfFile.createNewFile();
    testConf.writeXml(new FileOutputStream(testConfFile));
    testConfFile.deleteOnExit();
  } catch (IOException e) {
    // TODO Auto-generated catch block
    e.printStackTrace();
    throw new RuntimeException(e);
  }

  remoteFs.mkdirs(new Path("/tmp/input/"));
  remoteFs.mkdirs(new Path("/tmp/splitsDirNew/"));
  remoteFs.mkdirs(new Path("/tmp/splitsDirOld/"));
  testFilePath = remoteFs.makeQualified(new Path("/tmp/input/test.xml"));
  remoteFs.copyFromLocalFile(new Path(testConfFile.getAbsolutePath()),
      testFilePath);
  FileStatus fsStatus = remoteFs.getFileStatus(testFilePath);
  Assert.assertTrue(fsStatus.getLen() > 0);

  oldSplitsDir = remoteFs.makeQualified(new Path("/tmp/splitsDirOld/"));
  newSplitsDir = remoteFs.makeQualified(new Path("/tmp/splitsDirNew/"));
}
 
Example #23
Source File: GetGroupsForTesting.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
protected GetUserMappingsProtocol getUgmProtocol() throws IOException {
  Configuration conf = getConf();
  
  final InetSocketAddress addr = conf.getSocketAddr(
      YarnConfiguration.RM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_RM_ADMIN_PORT);
  final YarnRPC rpc = YarnRPC.create(conf);
  
  ResourceManagerAdministrationProtocol adminProtocol = (ResourceManagerAdministrationProtocol) rpc.getProxy(
      ResourceManagerAdministrationProtocol.class, addr, getConf());

  return adminProtocol;
}
 
Example #24
Source File: TestDockerContainerRuntime.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testCGroupParent() throws ContainerExecutionException {
  String hierarchy = "hadoop-yarn-test";
    conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY,
      hierarchy);

  DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime
      (mockExecutor, mockCGroupsHandler);
  runtime.initialize(conf);

  String resourceOptionsNone = "cgroups=none";
  DockerRunCommand command = Mockito.mock(DockerRunCommand.class);

  Mockito.when(mockCGroupsHandler.getRelativePathForCGroup(containerId))
      .thenReturn(hierarchy + "/" + containerIdStr);
  runtime.addCGroupParentIfRequired(resourceOptionsNone, containerIdStr,
      command);

  //no --cgroup-parent should be added here
  Mockito.verifyZeroInteractions(command);

  String resourceOptionsCpu = "/sys/fs/cgroup/cpu/" + hierarchy +
      containerIdStr;
  runtime.addCGroupParentIfRequired(resourceOptionsCpu, containerIdStr,
      command);

  //--cgroup-parent should be added for the containerId in question
  String expectedPath = "/" + hierarchy + "/" + containerIdStr;
  Mockito.verify(command).setCGroupParent(expectedPath);
}
 
Example #25
Source File: YarnAuthorizationProvider.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static YarnAuthorizationProvider getInstance(Configuration conf) {
  synchronized (YarnAuthorizationProvider.class) {
    if (authorizer == null) {
      Class<?> authorizerClass =
          conf.getClass(YarnConfiguration.YARN_AUTHORIZATION_PROVIDER,
            ConfiguredYarnAuthorizer.class);
      authorizer =
          (YarnAuthorizationProvider) ReflectionUtils.newInstance(
            authorizerClass, conf);
      authorizer.init(conf);
      LOG.info(authorizerClass.getName() + " is instiantiated.");
    }
  }
  return authorizer;
}
 
Example #26
Source File: TestNodeManagerShutdown.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testStateStoreRemovalOnDecommission() throws IOException {
  final File recoveryDir = new File(basedir, "nm-recovery");
  nm = new TestNodeManager();
  YarnConfiguration conf = createNMConfig();
  conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
  conf.set(YarnConfiguration.NM_RECOVERY_DIR, recoveryDir.getAbsolutePath());

  // verify state store is not removed on normal shutdown
  nm.init(conf);
  nm.start();
  Assert.assertTrue(recoveryDir.exists());
  Assert.assertTrue(recoveryDir.isDirectory());
  nm.stop();
  nm = null;
  Assert.assertTrue(recoveryDir.exists());
  Assert.assertTrue(recoveryDir.isDirectory());

  // verify state store is removed on decommissioned shutdown
  nm = new TestNodeManager();
  nm.init(conf);
  nm.start();
  Assert.assertTrue(recoveryDir.exists());
  Assert.assertTrue(recoveryDir.isDirectory());
  nm.getNMContext().setDecommissioned(true);
  nm.stop();
  nm = null;
  Assert.assertFalse(recoveryDir.exists());
}
 
Example #27
Source File: TestYarnService.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
@Test
public void testDistroMapRDefaults() throws Exception {
  assumeNonMaprProfile();

  YarnController controller = Mockito.mock(YarnController.class);
  YarnService yarnService = new YarnService(new TestListener(), controller, Mockito.mock(NodeProvider.class));

  properties.clear(MAPR_IMPALA_RA_THROTTLE);
  properties.clear(MAPR_MAX_RA_STREAMS);

  Cluster myCluster = createCluster();
  myCluster.getClusterConfig().setDistroType(DistroType.MAPR).setIsSecure(true);
  YarnConfiguration myYarnConfig = new YarnConfiguration();
  yarnService.updateYarnConfiguration(myCluster, myYarnConfig);

  assertEquals("/opt/mapr/conf/mapr.login.conf", myYarnConfig.get(YarnDefaultsConfigurator.JAVA_LOGIN));
  assertEquals("false", myYarnConfig.get(YarnDefaultsConfigurator.ZK_SASL_CLIENT));
  assertEquals("Client", myYarnConfig.get(YarnDefaultsConfigurator.ZK_SASL_CLIENT_CONFIG));
  assertEquals("com.mapr.security.maprsasl.MaprSaslProvider", myYarnConfig.get(YarnDefaultsConfigurator.ZK_SASL_PROVIDER));
  assertEquals("[\"maprfs:///var/mapr/local/${NM_HOST}/mapred/spill\"]", myYarnConfig.get(YarnDefaultsConfigurator
    .SPILL_PATH));
  assertEquals("0", myYarnConfig.get(NETTY_MAX_DIRECT_MEMORY));
  assertNull(myYarnConfig.get(MAPR_IMPALA_RA_THROTTLE));
  assertNull(myYarnConfig.get(MAPR_MAX_RA_STREAMS));

  Cluster myClusterOff = createCluster();
  myClusterOff.getClusterConfig().setDistroType(DistroType.MAPR).setIsSecure(false);
  YarnConfiguration myYarnConfigOff = new YarnConfiguration();
  yarnService.updateYarnConfiguration(myClusterOff, myYarnConfigOff);

  assertEquals("/opt/mapr/conf/mapr.login.conf", myYarnConfigOff.get(YarnDefaultsConfigurator.JAVA_LOGIN));
  assertEquals("false", myYarnConfigOff.get(YarnDefaultsConfigurator.ZK_SASL_CLIENT));
  assertEquals("Client_simple", myYarnConfigOff.get(YarnDefaultsConfigurator.ZK_SASL_CLIENT_CONFIG));
  assertEquals("com.mapr.security.simplesasl.SimpleSaslProvider", myYarnConfigOff.get(YarnDefaultsConfigurator.ZK_SASL_PROVIDER));
  assertEquals("[\"maprfs:///var/mapr/local/${NM_HOST}/mapred/spill\"]", myYarnConfigOff.get(YarnDefaultsConfigurator
    .SPILL_PATH));
  assertEquals("0", myYarnConfigOff.get(NETTY_MAX_DIRECT_MEMORY));
  assertNull(myYarnConfigOff.get(MAPR_IMPALA_RA_THROTTLE));
  assertNull(myYarnConfigOff.get(MAPR_MAX_RA_STREAMS));
}
 
Example #28
Source File: CliFrontendRunWithYarnTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public AbstractYarnClusterDescriptor createClusterDescriptor(CommandLine commandLine)
	throws FlinkException {
	AbstractYarnClusterDescriptor parent = super.createClusterDescriptor(commandLine);
	return new NonDeployingDetachedYarnClusterDescriptor(
			parent.getFlinkConfiguration(),
			(YarnConfiguration) parent.getYarnClient().getConfig(),
			configurationDirectory,
			parent.getYarnClient(),
			clusterClient);
}
 
Example #29
Source File: ApplicationHistoryClientService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  InetSocketAddress address = conf.getSocketAddr(
      YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
      YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT);

  Preconditions.checkArgument(conf.getInt(
      YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT) > 0,
      "%s property value should be greater than zero",
      YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT);

  server =
      rpc.getServer(ApplicationHistoryProtocol.class, this,
        address, conf, null, conf.getInt(
          YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT,
          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT));

  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
    refreshServiceAcls(conf, new TimelinePolicyProvider());
  }

  server.start();
  this.bindAddress =
      conf.updateConnectAddr(YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
                             YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
                             YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
                             server.getListenerAddress());
  LOG.info("Instantiated ApplicationHistoryClientService at "
      + this.bindAddress);

  super.serviceStart();
}
 
Example #30
Source File: ResourceManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected void createPolicyMonitors() {
  if (scheduler instanceof PreemptableResourceScheduler
      && conf.getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS)) {
    LOG.info("Loading policy monitors");
    List<SchedulingEditPolicy> policies = conf.getInstances(
        YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
        SchedulingEditPolicy.class);
    if (policies.size() > 0) {
      rmDispatcher.register(ContainerPreemptEventType.class,
          new RMContainerPreemptEventDispatcher(
              (PreemptableResourceScheduler) scheduler));
      for (SchedulingEditPolicy policy : policies) {
        LOG.info("LOADING SchedulingEditPolicy:" + policy.getPolicyName());
        // periodically check whether we need to take action to guarantee
        // constraints
        SchedulingMonitor mon = new SchedulingMonitor(rmContext, policy);
        addService(mon);
      }
    } else {
      LOG.warn("Policy monitors configured (" +
          YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS +
          ") but none specified (" +
          YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES + ")");
    }
  }
}