org.apache.hadoop.yarn.conf.YarnConfiguration Java Examples
The following examples show how to use
org.apache.hadoop.yarn.conf.YarnConfiguration.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestLocalDirsHandlerService.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testValidPathsDirHandlerService() throws Exception { Configuration conf = new YarnConfiguration(); String localDir1 = new File("file:///" + testDir, "localDir1").getPath(); String localDir2 = new File("hdfs:///" + testDir, "localDir2").getPath(); conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir1 + "," + localDir2); String logDir1 = new File("file:///" + testDir, "logDir1").getPath(); conf.set(YarnConfiguration.NM_LOG_DIRS, logDir1); LocalDirsHandlerService dirSvc = new LocalDirsHandlerService(); try { dirSvc.init(conf); Assert.fail("Service should have thrown an exception due to wrong URI"); } catch (YarnRuntimeException e) { } Assert.assertEquals("Service should not be inited", STATE.STOPPED, dirSvc.getServiceState()); dirSvc.close(); }
Example #2
Source File: AdminService.java From hadoop with Apache License 2.0 | 6 votes |
@Override public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings( RefreshUserToGroupsMappingsRequest request) throws YarnException, IOException { String argName = "refreshUserToGroupsMappings"; UserGroupInformation user = checkAcls(argName); checkRMStatus(user.getShortUserName(), argName, "refresh user-groups."); Groups.getUserToGroupsMappingService( getConfiguration(new Configuration(false), YarnConfiguration.CORE_SITE_CONFIGURATION_FILE)).refresh(); RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService"); return recordFactory.newRecordInstance( RefreshUserToGroupsMappingsResponse.class); }
Example #3
Source File: TestSharedCacheUploader.java From hadoop with Apache License 2.0 | 6 votes |
/** * If resource is public, verifyAccess should succeed */ @Test public void testVerifyAccessPublicResource() throws Exception { Configuration conf = new Configuration(); conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true); LocalResource resource = mock(LocalResource.class); // give public visibility when(resource.getVisibility()).thenReturn(LocalResourceVisibility.PUBLIC); Path localPath = mock(Path.class); when(localPath.getName()).thenReturn("foo.jar"); String user = "joe"; SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class); FileSystem fs = mock(FileSystem.class); FileSystem localFs = FileSystem.getLocal(conf); SharedCacheUploader spied = createSpiedUploader(resource, localPath, user, conf, scmClient, fs, localFs); assertTrue(spied.verifyAccess()); }
Example #4
Source File: TestResourceManager.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testNMExpiryAndHeartbeatIntervalsValidation() throws Exception { Configuration conf = new YarnConfiguration(); conf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 1000); conf.setLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 1001); resourceManager = new ResourceManager();; try { resourceManager.init(conf); } catch (YarnRuntimeException e) { // Exception is expected. if (!e.getMessage().startsWith("Nodemanager expiry interval should be no" + " less than heartbeat interval")) { throw e; } } }
Example #5
Source File: TestRMWebServices.java From big-c with Apache License 2.0 | 6 votes |
public void verifyClusterSchedulerFifoGeneric(String type, String state, float capacity, float usedCapacity, int minQueueCapacity, int maxQueueCapacity, int numNodes, int usedNodeCapacity, int availNodeCapacity, int totalNodeCapacity, int numContainers) throws JSONException, Exception { assertEquals("type doesn't match", "fifoScheduler", type); assertEquals("qstate doesn't match", QueueState.RUNNING.toString(), state); assertEquals("capacity doesn't match", 1.0, capacity, 0.0); assertEquals("usedCapacity doesn't match", 0.0, usedCapacity, 0.0); assertEquals( "minQueueMemoryCapacity doesn't match", YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, minQueueCapacity); assertEquals("maxQueueMemoryCapacity doesn't match", YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, maxQueueCapacity); assertEquals("numNodes doesn't match", 0, numNodes); assertEquals("usedNodeCapacity doesn't match", 0, usedNodeCapacity); assertEquals("availNodeCapacity doesn't match", 0, availNodeCapacity); assertEquals("totalNodeCapacity doesn't match", 0, totalNodeCapacity); assertEquals("numContainers doesn't match", 0, numContainers); }
Example #6
Source File: TestFairScheduler.java From hadoop with Apache License 2.0 | 6 votes |
@Test public void testMinZeroResourcesSettings() throws IOException { scheduler = new FairScheduler(); YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 0); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_GCORES, 0); conf.setInt( FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512); conf.setInt( FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 2); conf.setInt( FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_GCORES, 1); scheduler.init(conf); scheduler.reinitialize(conf, null); Assert.assertEquals(0, scheduler.getMinimumResourceCapability().getMemory()); Assert.assertEquals(0, scheduler.getMinimumResourceCapability().getVirtualCores()); Assert.assertEquals(0, scheduler.getMinimumResourceCapability().getGpuCores()); Assert.assertEquals(512, scheduler.getIncrementResourceCapability().getMemory()); Assert.assertEquals(2, scheduler.getIncrementResourceCapability().getVirtualCores()); Assert.assertEquals(1, scheduler.getIncrementResourceCapability().getGpuCores()); }
Example #7
Source File: TestFifoScheduler.java From hadoop with Apache License 2.0 | 6 votes |
private void testMinimumAllocation(YarnConfiguration conf, int testAlloc) throws Exception { MockRM rm = new MockRM(conf); rm.start(); // Register node1 MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); // Submit an application RMApp app1 = rm.submitApp(testAlloc); // kick the scheduling nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport( nm1.getNodeId()); int checkAlloc = conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); Assert.assertEquals(checkAlloc, report_nm1.getUsedResource().getMemory()); rm.stop(); }
Example #8
Source File: TestSharedCacheUploader.java From hadoop with Apache License 2.0 | 6 votes |
/** * If the localPath does not exists, getActualPath should get to one level * down */ @Test public void testGetActualPath() throws Exception { Configuration conf = new Configuration(); conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true); LocalResource resource = mock(LocalResource.class); // give public visibility when(resource.getVisibility()).thenReturn(LocalResourceVisibility.PUBLIC); Path localPath = new Path("foo.jar"); String user = "joe"; SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class); FileSystem fs = mock(FileSystem.class); FileSystem localFs = mock(FileSystem.class); // stub it to return a status that indicates a directory FileStatus status = mock(FileStatus.class); when(status.isDirectory()).thenReturn(true); when(localFs.getFileStatus(localPath)).thenReturn(status); SharedCacheUploader spied = createSpiedUploader(resource, localPath, user, conf, scmClient, fs, localFs); Path actualPath = spied.getActualPath(); assertEquals(actualPath.getName(), localPath.getName()); assertEquals(actualPath.getParent().getName(), localPath.getName()); }
Example #9
Source File: TestYarnService.java From dremio-oss with Apache License 2.0 | 6 votes |
@Test public void testDistroDefaultsOverwrite() throws Exception { assumeNonMaprProfile(); YarnController controller = Mockito.mock(YarnController.class); YarnService yarnService = new YarnService(new TestListener(), controller, Mockito.mock(NodeProvider.class)); Cluster myCluster = createCluster(); List<Property> props = myCluster.getClusterConfig().getSubPropertyList(); props.add(new Property(YarnDefaultsConfigurator.SPILL_PATH, "/abc/bcd")); props.add(new Property(YarnDefaultsConfigurator.JAVA_LOGIN, "/abc/bcd/login.conf")); myCluster.getClusterConfig().setDistroType(DistroType.HDP).setIsSecure(true); YarnConfiguration myYarnConfig = new YarnConfiguration(); yarnService.updateYarnConfiguration(myCluster, myYarnConfig); assertEquals("/abc/bcd/login.conf", myYarnConfig.get(YarnDefaultsConfigurator.JAVA_LOGIN)); assertNull(myYarnConfig.get(YarnDefaultsConfigurator.ZK_SASL_CLIENT)); assertNull(myYarnConfig.get(YarnDefaultsConfigurator.ZK_SASL_CLIENT_CONFIG)); assertNull(myYarnConfig.get(YarnDefaultsConfigurator.ZK_SASL_PROVIDER)); assertEquals("/abc/bcd", myYarnConfig.get(YarnDefaultsConfigurator.SPILL_PATH)); }
Example #10
Source File: TestWorkPreservingRMRestart.java From big-c with Apache License 2.0 | 6 votes |
/** * Test validateAndCreateResourceRequest fails on recovery, app should ignore * this Exception and continue */ @Test (timeout = 30000) public void testAppFailToValidateResourceRequestOnRecovery() throws Exception{ MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.init(conf); rm1 = new MockRM(conf, memStore); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService()); nm1.registerNode(); RMApp app1 = rm1.submitApp(200); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // Change the config so that validateAndCreateResourceRequest throws // exception on recovery conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 50); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 100); rm2 = new MockRM(conf, memStore); nm1.setResourceTrackerService(rm2.getResourceTrackerService()); rm2.start(); }
Example #11
Source File: FileSystemRMStateStore.java From hadoop with Apache License 2.0 | 6 votes |
@Override protected synchronized void startInternal() throws Exception { // create filesystem only now, as part of service-start. By this time, RM is // authenticated with kerberos so we are good to create a file-system // handle. Configuration conf = new Configuration(getConfig()); conf.setBoolean("dfs.client.retry.policy.enabled", true); String retryPolicy = conf.get(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC, YarnConfiguration.DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC); conf.set("dfs.client.retry.policy.spec", retryPolicy); fs = fsWorkingPath.getFileSystem(conf); mkdirsWithRetries(rmDTSecretManagerRoot); mkdirsWithRetries(rmAppRoot); mkdirsWithRetries(amrmTokenSecretManagerRoot); }
Example #12
Source File: TestRMFailover.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testExplicitFailover() throws YarnException, InterruptedException, IOException { conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); cluster.init(conf); cluster.start(); getAdminService(0).transitionToActive(req); assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex()); verifyConnections(); explicitFailover(); verifyConnections(); explicitFailover(); verifyConnections(); }
Example #13
Source File: TestSharedCacheUploaderService.java From big-c with Apache License 2.0 | 6 votes |
@Before public void startUp() { Configuration conf = new Configuration(); conf.set(YarnConfiguration.SCM_STORE_CLASS, InMemorySCMStore.class.getName()); conf.set(YarnConfiguration.SHARED_CACHE_ROOT, testDir.getPath()); AppChecker appChecker = spy(new DummyAppChecker()); store = new InMemorySCMStore(appChecker); store.init(conf); store.start(); service = new SharedCacheUploaderService(store); service.init(conf); service.start(); YarnRPC rpc = YarnRPC.create(new Configuration()); InetSocketAddress scmAddress = conf.getSocketAddr(YarnConfiguration.SCM_UPLOADER_SERVER_ADDRESS, YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_ADDRESS, YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_PORT); proxy = (SCMUploaderProtocol) rpc.getProxy( SCMUploaderProtocol.class, scmAddress, conf); }
Example #14
Source File: TestTimelineACLsManager.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testYarnACLsEnabledForDomain() throws Exception { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); TimelineACLsManager timelineACLsManager = new TimelineACLsManager(conf); TimelineDomain domain = new TimelineDomain(); domain.setOwner("owner"); Assert.assertTrue( "Owner should be allowed to access", timelineACLsManager.checkAccess( UserGroupInformation.createRemoteUser("owner"), domain)); Assert.assertFalse( "Other shouldn't be allowed to access", timelineACLsManager.checkAccess( UserGroupInformation.createRemoteUser("other"), domain)); Assert.assertTrue( "Admin should be allowed to access", timelineACLsManager.checkAccess( UserGroupInformation.createRemoteUser("admin"), domain)); }
Example #15
Source File: YarnTestBase.java From flink with Apache License 2.0 | 6 votes |
public static void populateYarnSecureConfigurations(Configuration conf, String principal, String keytab) { conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, "true"); conf.set(YarnConfiguration.RM_KEYTAB, keytab); conf.set(YarnConfiguration.RM_PRINCIPAL, principal); conf.set(YarnConfiguration.NM_KEYTAB, keytab); conf.set(YarnConfiguration.NM_PRINCIPAL, principal); conf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY, principal); conf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY, keytab); conf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY, principal); conf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY, keytab); conf.set("hadoop.security.auth_to_local", "RULE:[1:$1] RULE:[2:$1]"); }
Example #16
Source File: ConfiguredRMFailoverProxyProvider.java From big-c with Apache License 2.0 | 6 votes |
@Override public void init(Configuration configuration, RMProxy<T> rmProxy, Class<T> protocol) { this.rmProxy = rmProxy; this.protocol = protocol; this.rmProxy.checkAllowedProtocols(this.protocol); this.conf = new YarnConfiguration(configuration); Collection<String> rmIds = HAUtil.getRMHAIds(conf); this.rmServiceIds = rmIds.toArray(new String[rmIds.size()]); conf.set(YarnConfiguration.RM_HA_ID, rmServiceIds[currentProxyIndex]); conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES, YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES)); conf.setInt(CommonConfigurationKeysPublic. IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS, YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS)); }
Example #17
Source File: Client.java From Scribengin with GNU Affero General Public License v3.0 | 6 votes |
private void setupAppMasterEnv(Map<String, String> appMasterEnv) { StringBuilder classPathEnv = new StringBuilder(); classPathEnv.append(Environment.CLASSPATH.$()).append(File.pathSeparatorChar); classPathEnv.append("./*"); for (String c : conf.getStrings( YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { classPathEnv.append(File.pathSeparatorChar); classPathEnv.append(c.trim()); } String envStr = classPathEnv.toString(); LOG.info("env: " + envStr); appMasterEnv.put(Environment.CLASSPATH.name(), envStr); }
Example #18
Source File: TestResourceTrackerService.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testNodeRegistrationVersionLessThanRM() throws Exception { writeToHostsFile("host2"); Configuration conf = new Configuration(); conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile .getAbsolutePath()); conf.set(YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION,"EqualToRM" ); rm = new MockRM(conf); rm.start(); String nmVersion = "1.9.9"; ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService(); RegisterNodeManagerRequest req = Records.newRecord( RegisterNodeManagerRequest.class); NodeId nodeId = NodeId.newInstance("host2", 1234); Resource capability = BuilderUtils.newResource(1024, 1, 1); req.setResource(capability); req.setNodeId(nodeId); req.setHttpPort(1234); req.setNMVersion(nmVersion); // trying to register a invalid node. RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN,response.getNodeAction()); Assert.assertTrue("Diagnostic message did not contain: 'Disallowed NodeManager " + "Version "+ nmVersion + ", is less than the minimum version'", response.getDiagnosticsMessage().contains("Disallowed NodeManager Version " + nmVersion + ", is less than the minimum version ")); }
Example #19
Source File: NodeManager.java From hadoop with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws IOException { Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); StringUtils.startupShutdownMessage(NodeManager.class, args, LOG); NodeManager nodeManager = new NodeManager(); Configuration conf = new YarnConfiguration(); new GenericOptionsParser(conf, args); nodeManager.initAndStartNodeManager(conf, false); }
Example #20
Source File: TestDockerContainerExecutorWithMocks.java From big-c with Apache License 2.0 | 5 votes |
@Test(expected = IllegalArgumentException.class) public void testContainerLaunchInvalidImage() throws IOException { String appSubmitter = "nobody"; String appId = "APP_ID"; String containerId = "CONTAINER_ID"; String testImage = "testrepo.com/test-image rm -rf $HADOOP_PREFIX/*"; Container container = mock(Container.class, RETURNS_DEEP_STUBS); ContainerId cId = mock(ContainerId.class, RETURNS_DEEP_STUBS); ContainerLaunchContext context = mock(ContainerLaunchContext.class); HashMap<String, String> env = new HashMap<String,String>(); when(container.getContainerId()).thenReturn(cId); when(container.getLaunchContext()).thenReturn(context); when(cId.getApplicationAttemptId().getApplicationId().toString()).thenReturn(appId); when(cId.toString()).thenReturn(containerId); when(context.getEnvironment()).thenReturn(env); env.put(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage); dockerContainerExecutor.getConf() .set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage); Path scriptPath = new Path("file:///bin/echo"); Path tokensPath = new Path("file:///dev/null"); Path pidFile = new Path(workDir, "pid.txt"); dockerContainerExecutor.activateContainer(cId, pidFile); dockerContainerExecutor.launchContainer(container, scriptPath, tokensPath, appSubmitter, appId, workDir, dirsHandler.getLocalDirs(), dirsHandler.getLogDirs()); }
Example #21
Source File: TezClient.java From tez with Apache License 2.0 | 5 votes |
/** * Alternative to start() that explicitly sets sessionAppId and doesn't start a new AM. * The caller of getClient is responsible for initializing the new TezClient with a * Configuration compatible with the existing AM. It is expected the caller has cached the * original Configuration (e.g. in Zookeeper). * * In contrast to "start", no resources are localized. It is the responsibility of the caller to * ensure that existing localized resources and staging dirs are still valid. * * @param appId * @return 'this' just as a convenience for fluent style chaining */ public synchronized TezClient getClient(ApplicationId appId) throws TezException, IOException { sessionAppId = appId; amConfig.setYarnConfiguration(new YarnConfiguration(amConfig.getTezConfiguration())); startFrameworkClient(); setupJavaOptsChecker(); if (!isSession) { String msg = "Must be in session mode to bind TezClient to existing AM"; LOG.error(msg); throw new IllegalStateException(msg); } LOG.info("Session mode. Reconnecting to session: " + sessionAppId.toString()); clientTimeout = amConfig.getTezConfiguration().getInt( TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS, TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS_DEFAULT); try { setupApplicationContext(); ApplicationReport appReport = frameworkClient.getApplicationReport(sessionAppId); LOG.info("The url to track the Tez Session: " + appReport.getTrackingUrl()); sessionStarted.set(true); } catch (YarnException e) { throw new TezException(e); } startClientHeartbeat(); this.stagingFs = FileSystem.get(amConfig.getTezConfiguration()); return this; }
Example #22
Source File: TestMRHelpers.java From incubator-tez with Apache License 2.0 | 5 votes |
@BeforeClass public static void setup() throws IOException { try { conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR); dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2) .format(true).racks(null).build(); remoteFs = dfsCluster.getFileSystem(); } catch (IOException io) { throw new RuntimeException("problem starting mini dfs cluster", io); } Configuration testConf = new YarnConfiguration( dfsCluster.getFileSystem().getConf()); File testConfFile = new File(TEST_ROOT_DIR, "test.xml"); try { testConfFile.createNewFile(); testConf.writeXml(new FileOutputStream(testConfFile)); testConfFile.deleteOnExit(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); throw new RuntimeException(e); } remoteFs.mkdirs(new Path("/tmp/input/")); remoteFs.mkdirs(new Path("/tmp/splitsDirNew/")); remoteFs.mkdirs(new Path("/tmp/splitsDirOld/")); testFilePath = remoteFs.makeQualified(new Path("/tmp/input/test.xml")); remoteFs.copyFromLocalFile(new Path(testConfFile.getAbsolutePath()), testFilePath); FileStatus fsStatus = remoteFs.getFileStatus(testFilePath); Assert.assertTrue(fsStatus.getLen() > 0); oldSplitsDir = remoteFs.makeQualified(new Path("/tmp/splitsDirOld/")); newSplitsDir = remoteFs.makeQualified(new Path("/tmp/splitsDirNew/")); }
Example #23
Source File: GetGroupsForTesting.java From hadoop with Apache License 2.0 | 5 votes |
@Override protected GetUserMappingsProtocol getUgmProtocol() throws IOException { Configuration conf = getConf(); final InetSocketAddress addr = conf.getSocketAddr( YarnConfiguration.RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_PORT); final YarnRPC rpc = YarnRPC.create(conf); ResourceManagerAdministrationProtocol adminProtocol = (ResourceManagerAdministrationProtocol) rpc.getProxy( ResourceManagerAdministrationProtocol.class, addr, getConf()); return adminProtocol; }
Example #24
Source File: TestDockerContainerRuntime.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testCGroupParent() throws ContainerExecutionException { String hierarchy = "hadoop-yarn-test"; conf.set(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_HIERARCHY, hierarchy); DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime (mockExecutor, mockCGroupsHandler); runtime.initialize(conf); String resourceOptionsNone = "cgroups=none"; DockerRunCommand command = Mockito.mock(DockerRunCommand.class); Mockito.when(mockCGroupsHandler.getRelativePathForCGroup(containerId)) .thenReturn(hierarchy + "/" + containerIdStr); runtime.addCGroupParentIfRequired(resourceOptionsNone, containerIdStr, command); //no --cgroup-parent should be added here Mockito.verifyZeroInteractions(command); String resourceOptionsCpu = "/sys/fs/cgroup/cpu/" + hierarchy + containerIdStr; runtime.addCGroupParentIfRequired(resourceOptionsCpu, containerIdStr, command); //--cgroup-parent should be added for the containerId in question String expectedPath = "/" + hierarchy + "/" + containerIdStr; Mockito.verify(command).setCGroupParent(expectedPath); }
Example #25
Source File: YarnAuthorizationProvider.java From hadoop with Apache License 2.0 | 5 votes |
public static YarnAuthorizationProvider getInstance(Configuration conf) { synchronized (YarnAuthorizationProvider.class) { if (authorizer == null) { Class<?> authorizerClass = conf.getClass(YarnConfiguration.YARN_AUTHORIZATION_PROVIDER, ConfiguredYarnAuthorizer.class); authorizer = (YarnAuthorizationProvider) ReflectionUtils.newInstance( authorizerClass, conf); authorizer.init(conf); LOG.info(authorizerClass.getName() + " is instiantiated."); } } return authorizer; }
Example #26
Source File: TestNodeManagerShutdown.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testStateStoreRemovalOnDecommission() throws IOException { final File recoveryDir = new File(basedir, "nm-recovery"); nm = new TestNodeManager(); YarnConfiguration conf = createNMConfig(); conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true); conf.set(YarnConfiguration.NM_RECOVERY_DIR, recoveryDir.getAbsolutePath()); // verify state store is not removed on normal shutdown nm.init(conf); nm.start(); Assert.assertTrue(recoveryDir.exists()); Assert.assertTrue(recoveryDir.isDirectory()); nm.stop(); nm = null; Assert.assertTrue(recoveryDir.exists()); Assert.assertTrue(recoveryDir.isDirectory()); // verify state store is removed on decommissioned shutdown nm = new TestNodeManager(); nm.init(conf); nm.start(); Assert.assertTrue(recoveryDir.exists()); Assert.assertTrue(recoveryDir.isDirectory()); nm.getNMContext().setDecommissioned(true); nm.stop(); nm = null; Assert.assertFalse(recoveryDir.exists()); }
Example #27
Source File: TestYarnService.java From dremio-oss with Apache License 2.0 | 5 votes |
@Test public void testDistroMapRDefaults() throws Exception { assumeNonMaprProfile(); YarnController controller = Mockito.mock(YarnController.class); YarnService yarnService = new YarnService(new TestListener(), controller, Mockito.mock(NodeProvider.class)); properties.clear(MAPR_IMPALA_RA_THROTTLE); properties.clear(MAPR_MAX_RA_STREAMS); Cluster myCluster = createCluster(); myCluster.getClusterConfig().setDistroType(DistroType.MAPR).setIsSecure(true); YarnConfiguration myYarnConfig = new YarnConfiguration(); yarnService.updateYarnConfiguration(myCluster, myYarnConfig); assertEquals("/opt/mapr/conf/mapr.login.conf", myYarnConfig.get(YarnDefaultsConfigurator.JAVA_LOGIN)); assertEquals("false", myYarnConfig.get(YarnDefaultsConfigurator.ZK_SASL_CLIENT)); assertEquals("Client", myYarnConfig.get(YarnDefaultsConfigurator.ZK_SASL_CLIENT_CONFIG)); assertEquals("com.mapr.security.maprsasl.MaprSaslProvider", myYarnConfig.get(YarnDefaultsConfigurator.ZK_SASL_PROVIDER)); assertEquals("[\"maprfs:///var/mapr/local/${NM_HOST}/mapred/spill\"]", myYarnConfig.get(YarnDefaultsConfigurator .SPILL_PATH)); assertEquals("0", myYarnConfig.get(NETTY_MAX_DIRECT_MEMORY)); assertNull(myYarnConfig.get(MAPR_IMPALA_RA_THROTTLE)); assertNull(myYarnConfig.get(MAPR_MAX_RA_STREAMS)); Cluster myClusterOff = createCluster(); myClusterOff.getClusterConfig().setDistroType(DistroType.MAPR).setIsSecure(false); YarnConfiguration myYarnConfigOff = new YarnConfiguration(); yarnService.updateYarnConfiguration(myClusterOff, myYarnConfigOff); assertEquals("/opt/mapr/conf/mapr.login.conf", myYarnConfigOff.get(YarnDefaultsConfigurator.JAVA_LOGIN)); assertEquals("false", myYarnConfigOff.get(YarnDefaultsConfigurator.ZK_SASL_CLIENT)); assertEquals("Client_simple", myYarnConfigOff.get(YarnDefaultsConfigurator.ZK_SASL_CLIENT_CONFIG)); assertEquals("com.mapr.security.simplesasl.SimpleSaslProvider", myYarnConfigOff.get(YarnDefaultsConfigurator.ZK_SASL_PROVIDER)); assertEquals("[\"maprfs:///var/mapr/local/${NM_HOST}/mapred/spill\"]", myYarnConfigOff.get(YarnDefaultsConfigurator .SPILL_PATH)); assertEquals("0", myYarnConfigOff.get(NETTY_MAX_DIRECT_MEMORY)); assertNull(myYarnConfigOff.get(MAPR_IMPALA_RA_THROTTLE)); assertNull(myYarnConfigOff.get(MAPR_MAX_RA_STREAMS)); }
Example #28
Source File: CliFrontendRunWithYarnTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public AbstractYarnClusterDescriptor createClusterDescriptor(CommandLine commandLine) throws FlinkException { AbstractYarnClusterDescriptor parent = super.createClusterDescriptor(commandLine); return new NonDeployingDetachedYarnClusterDescriptor( parent.getFlinkConfiguration(), (YarnConfiguration) parent.getYarnClient().getConfig(), configurationDirectory, parent.getYarnClient(), clusterClient); }
Example #29
Source File: ApplicationHistoryClientService.java From hadoop with Apache License 2.0 | 5 votes |
protected void serviceStart() throws Exception { Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); InetSocketAddress address = conf.getSocketAddr( YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, YarnConfiguration.TIMELINE_SERVICE_ADDRESS, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT); Preconditions.checkArgument(conf.getInt( YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT) > 0, "%s property value should be greater than zero", YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT); server = rpc.getServer(ApplicationHistoryProtocol.class, this, address, conf, null, conf.getInt( YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT)); // Enable service authorization? if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { refreshServiceAcls(conf, new TimelinePolicyProvider()); } server.start(); this.bindAddress = conf.updateConnectAddr(YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, YarnConfiguration.TIMELINE_SERVICE_ADDRESS, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS, server.getListenerAddress()); LOG.info("Instantiated ApplicationHistoryClientService at " + this.bindAddress); super.serviceStart(); }
Example #30
Source File: ResourceManager.java From big-c with Apache License 2.0 | 5 votes |
protected void createPolicyMonitors() { if (scheduler instanceof PreemptableResourceScheduler && conf.getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS)) { LOG.info("Loading policy monitors"); List<SchedulingEditPolicy> policies = conf.getInstances( YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES, SchedulingEditPolicy.class); if (policies.size() > 0) { rmDispatcher.register(ContainerPreemptEventType.class, new RMContainerPreemptEventDispatcher( (PreemptableResourceScheduler) scheduler)); for (SchedulingEditPolicy policy : policies) { LOG.info("LOADING SchedulingEditPolicy:" + policy.getPolicyName()); // periodically check whether we need to take action to guarantee // constraints SchedulingMonitor mon = new SchedulingMonitor(rmContext, policy); addService(mon); } } else { LOG.warn("Policy monitors configured (" + YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS + ") but none specified (" + YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES + ")"); } } }