org.apache.hadoop.yarn.server.resourcemanager.ClientRMService Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.ClientRMService. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestDelegationTokenRenewer.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  counter = new AtomicInteger(0);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  eventQueue = new LinkedBlockingQueue<Event>();
  dispatcher = new AsyncDispatcher(eventQueue);
  Renewer.reset();
  delegationTokenRenewer = createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext =  mock(RMContext.class);
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      delegationTokenRenewer);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  delegationTokenRenewer.setRMContext(mockContext);
  delegationTokenRenewer.init(conf);
  delegationTokenRenewer.start();
}
 
Example #2
Source File: TestDelegationTokenRenewer.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  counter = new AtomicInteger(0);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  eventQueue = new LinkedBlockingQueue<Event>();
  dispatcher = new AsyncDispatcher(eventQueue);
  Renewer.reset();
  delegationTokenRenewer = createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext =  mock(RMContext.class);
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      delegationTokenRenewer);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  delegationTokenRenewer.setRMContext(mockContext);
  delegationTokenRenewer.init(conf);
  delegationTokenRenewer.start();
}
 
Example #3
Source File: ProtocolHATestBase.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
protected ResourceManager createResourceManager() {
  return new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
      // Don't try to login using keytab in the testcases.
    }
    @Override
    protected ClientRMService createClientRMService() {
      if (overrideClientRMService) {
        return new CustomedClientRMService(this.rmContext, this.scheduler,
            this.rmAppManager, this.applicationACLsManager,
            this.queueACLsManager,
            this.rmContext.getRMDelegationTokenSecretManager());
      }
      return super.createClientRMService();
    }
    @Override
    protected ResourceTrackerService createResourceTrackerService() {
      if (overrideRTS) {
        return new CustomedResourceTrackerService(this.rmContext,
            this.nodesListManager, this.nmLivelinessMonitor,
            this.rmContext.getContainerTokenSecretManager(),
            this.rmContext.getNMTokenSecretManager());
      }
      return super.createResourceTrackerService();
    }
    @Override
    protected ApplicationMasterService createApplicationMasterService() {
      if (overrideApplicationMasterService) {
        return new CustomedApplicationMasterService(this.rmContext,
            this.scheduler);
      }
      return super.createApplicationMasterService();
    }
  };
}
 
Example #4
Source File: TestRMWebAppFairScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static ResourceManager mockRm(RMContext rmContext) throws
    IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs = mockFairScheduler();
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 
Example #5
Source File: TestRMWebApp.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static ClientRMService mockClientRMService(RMContext rmContext) {
  ClientRMService clientRMService = mock(ClientRMService.class);
  List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
  for (RMApp app : rmContext.getRMApps().values()) {
    ApplicationReport appReport =
        ApplicationReport.newInstance(
            app.getApplicationId(), (ApplicationAttemptId) null,
            app.getUser(), app.getQueue(),
            app.getName(), (String) null, 0, (Token) null,
            app.createApplicationState(),
            app.getDiagnostics().toString(), (String) null,
            app.getStartTime(), app.getFinishTime(),
            app.getFinalApplicationStatus(),
            (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
            app.getProgress(), app.getApplicationType(), (Token) null);
    appReports.add(appReport);
  }
  GetApplicationsResponse response = mock(GetApplicationsResponse.class);
  when(response.getApplicationList()).thenReturn(appReports);
  try {
    when(clientRMService.getApplications(any(GetApplicationsRequest.class)))
        .thenReturn(response);
  } catch (YarnException e) {
    Assert.fail("Exception is not expteced.");
  }
  return clientRMService;
}
 
Example #6
Source File: TestRMWebApp.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static ResourceManager mockRm(RMContext rmContext) throws IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs = mockCapacityScheduler();
  ApplicationACLsManager aclMgr = mockAppACLsManager();
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getApplicationACLsManager()).thenReturn(aclMgr);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 
Example #7
Source File: TestRMWebServices.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testAppsRace() throws Exception {
  // mock up an RM that returns app reports for apps that don't exist
  // in the RMApps list
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationReport mockReport = mock(ApplicationReport.class);
  when(mockReport.getApplicationId()).thenReturn(appId);
  GetApplicationsResponse mockAppsResponse =
      mock(GetApplicationsResponse.class);
  when(mockAppsResponse.getApplicationList())
    .thenReturn(Arrays.asList(new ApplicationReport[] { mockReport }));
  ClientRMService mockClientSvc = mock(ClientRMService.class);
  when(mockClientSvc.getApplications(isA(GetApplicationsRequest.class),
      anyBoolean())).thenReturn(mockAppsResponse);
  ResourceManager mockRM = mock(ResourceManager.class);
  RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
      null, null, null, null, null);
  when(mockRM.getRMContext()).thenReturn(rmContext);
  when(mockRM.getClientRMService()).thenReturn(mockClientSvc);

  RMWebServices webSvc = new RMWebServices(mockRM, new Configuration(),
      mock(HttpServletResponse.class));

  final Set<String> emptySet =
      Collections.unmodifiableSet(Collections.<String>emptySet());

  // verify we don't get any apps when querying
  HttpServletRequest mockHsr = mock(HttpServletRequest.class);
  AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
      null, null, null, null, null, null, null, emptySet, emptySet);
  assertTrue(appsInfo.getApps().isEmpty());

  // verify we don't get an NPE when specifying a final status query
  appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED",
      null, null, null, null, null, null, null, emptySet, emptySet);
  assertTrue(appsInfo.getApps().isEmpty());
}
 
Example #8
Source File: TestDelegationTokenRenewerLifecycle.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testStartupFailure() throws Exception {
  Configuration conf = new Configuration();
  DelegationTokenRenewer delegationTokenRenewer =
      new DelegationTokenRenewer();
  RMContext mockContext = mock(RMContext.class);
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  delegationTokenRenewer.setRMContext(mockContext);
  delegationTokenRenewer.init(conf);
  delegationTokenRenewer.stop();
}
 
Example #9
Source File: TestRMWebAppFairScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static ResourceManager mockRmWithApps(RMContext rmContext) throws
    IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs =  mockFairSchedulerWithoutApps(rmContext);
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 
Example #10
Source File: TestDelegationTokenRenewer.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testDTRonAppSubmission()
    throws IOException, InterruptedException, BrokenBarrierException {
  final Credentials credsx = new Credentials();
  final Token<DelegationTokenIdentifier> tokenx = mock(Token.class);
  when(tokenx.getKind()).thenReturn(new Text("HDFS_DELEGATION_TOKEN"));
  DelegationTokenIdentifier dtId1 = 
      new DelegationTokenIdentifier(new Text("user1"), new Text("renewer"),
        new Text("user1"));
  when(tokenx.decodeIdentifier()).thenReturn(dtId1);
  credsx.addToken(new Text("token"), tokenx);
  doReturn(true).when(tokenx).isManaged();
  doThrow(new IOException("boom"))
      .when(tokenx).renew(any(Configuration.class));
    // fire up the renewer
  final DelegationTokenRenewer dtr =
       createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext = mock(RMContext.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  dtr.setRMContext(mockContext);
  when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr);
  dtr.init(conf);
  dtr.start();

  try {
    dtr.addApplicationSync(mock(ApplicationId.class), credsx, false, "user");
    fail("Catch IOException on app submission");
  } catch (IOException e){
    Assert.assertTrue(e.getMessage().contains(tokenx.toString()));
    Assert.assertTrue(e.getCause().toString().contains("boom"));
  }

}
 
Example #11
Source File: TwillTester.java    From twill with Apache License 2.0 5 votes vote down vote up
public ApplicationResourceUsageReport getApplicationResourceReport(String appId) throws Exception {
  List<String> splits = Lists.newArrayList(Splitter.on('_').split(appId));
  Preconditions.checkArgument(splits.size() == 3, "Invalid application id - " + appId);
  ApplicationId applicationId = ApplicationId.newInstance(Long.parseLong(splits.get(1)),
                                                          Integer.parseInt(splits.get(2)));

  ClientRMService clientRMService = cluster.getResourceManager().getClientRMService();
  GetApplicationReportRequest request = Records.newRecord(GetApplicationReportRequest.class);
  request.setApplicationId(applicationId);
  return clientRMService.getApplicationReport(request)
    .getApplicationReport().getApplicationResourceUsageReport();
}
 
Example #12
Source File: ProtocolHATestBase.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected ResourceManager createResourceManager() {
  return new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
      // Don't try to login using keytab in the testcases.
    }
    @Override
    protected ClientRMService createClientRMService() {
      if (overrideClientRMService) {
        return new CustomedClientRMService(this.rmContext, this.scheduler,
            this.rmAppManager, this.applicationACLsManager,
            this.queueACLsManager,
            this.rmContext.getRMDelegationTokenSecretManager());
      }
      return super.createClientRMService();
    }
    @Override
    protected ResourceTrackerService createResourceTrackerService() {
      if (overrideRTS) {
        return new CustomedResourceTrackerService(this.rmContext,
            this.nodesListManager, this.nmLivelinessMonitor,
            this.rmContext.getContainerTokenSecretManager(),
            this.rmContext.getNMTokenSecretManager());
      }
      return super.createResourceTrackerService();
    }
    @Override
    protected ApplicationMasterService createApplicationMasterService() {
      if (overrideApplicationMasterService) {
        return new CustomedApplicationMasterService(this.rmContext,
            this.scheduler);
      }
      return super.createApplicationMasterService();
    }
  };
}
 
Example #13
Source File: TestRMWebAppFairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static ResourceManager mockRmWithApps(RMContext rmContext) throws
    IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs =  mockFairSchedulerWithoutApps(rmContext);
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 
Example #14
Source File: TestRMWebAppFairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static ResourceManager mockRm(RMContext rmContext) throws
    IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs = mockFairScheduler();
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 
Example #15
Source File: TestRMWebApp.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static ClientRMService mockClientRMService(RMContext rmContext) {
  ClientRMService clientRMService = mock(ClientRMService.class);
  List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
  for (RMApp app : rmContext.getRMApps().values()) {
    ApplicationReport appReport =
        ApplicationReport.newInstance(
            app.getApplicationId(), (ApplicationAttemptId) null,
            app.getUser(), app.getQueue(),
            app.getName(), (String) null, 0, (Token) null,
            app.createApplicationState(),
            app.getDiagnostics().toString(), (String) null,
            app.getStartTime(), app.getFinishTime(),
            app.getFinalApplicationStatus(),
            (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
            app.getProgress(), app.getApplicationType(), (Token) null);
    appReports.add(appReport);
  }
  GetApplicationsResponse response = mock(GetApplicationsResponse.class);
  when(response.getApplicationList()).thenReturn(appReports);
  try {
    when(clientRMService.getApplications(any(GetApplicationsRequest.class)))
        .thenReturn(response);
  } catch (YarnException e) {
    Assert.fail("Exception is not expteced.");
  }
  return clientRMService;
}
 
Example #16
Source File: TestRMWebApp.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static ResourceManager mockRm(RMContext rmContext) throws IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs = mockCapacityScheduler();
  ApplicationACLsManager aclMgr = mockAppACLsManager();
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getApplicationACLsManager()).thenReturn(aclMgr);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 
Example #17
Source File: TestRMWebServices.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testAppsRace() throws Exception {
  // mock up an RM that returns app reports for apps that don't exist
  // in the RMApps list
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationReport mockReport = mock(ApplicationReport.class);
  when(mockReport.getApplicationId()).thenReturn(appId);
  GetApplicationsResponse mockAppsResponse =
      mock(GetApplicationsResponse.class);
  when(mockAppsResponse.getApplicationList())
    .thenReturn(Arrays.asList(new ApplicationReport[] { mockReport }));
  ClientRMService mockClientSvc = mock(ClientRMService.class);
  when(mockClientSvc.getApplications(isA(GetApplicationsRequest.class),
      anyBoolean())).thenReturn(mockAppsResponse);
  ResourceManager mockRM = mock(ResourceManager.class);
  RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
      null, null, null, null, null);
  when(mockRM.getRMContext()).thenReturn(rmContext);
  when(mockRM.getClientRMService()).thenReturn(mockClientSvc);

  RMWebServices webSvc = new RMWebServices(mockRM, new Configuration(),
      mock(HttpServletResponse.class));

  final Set<String> emptySet =
      Collections.unmodifiableSet(Collections.<String>emptySet());

  // verify we don't get any apps when querying
  HttpServletRequest mockHsr = mock(HttpServletRequest.class);
  AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
      null, null, null, null, null, null, null, emptySet, emptySet);
  assertTrue(appsInfo.getApps().isEmpty());

  // verify we don't get an NPE when specifying a final status query
  appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED",
      null, null, null, null, null, null, null, emptySet, emptySet);
  assertTrue(appsInfo.getApps().isEmpty());
}
 
Example #18
Source File: TestDelegationTokenRenewerLifecycle.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testStartupFailure() throws Exception {
  Configuration conf = new Configuration();
  DelegationTokenRenewer delegationTokenRenewer =
      new DelegationTokenRenewer();
  RMContext mockContext = mock(RMContext.class);
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  delegationTokenRenewer.setRMContext(mockContext);
  delegationTokenRenewer.init(conf);
  delegationTokenRenewer.stop();
}
 
Example #19
Source File: StramMiniClusterTest.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
private void checkNodeState() throws YarnException
{
  GetClusterNodesRequest request = Records.newRecord(GetClusterNodesRequest.class);
  ClientRMService clientRMService = yarnCluster.getResourceManager().getClientRMService();
  GetClusterNodesResponse response = clientRMService.getClusterNodes(request);
  List<NodeReport> nodeReports = response.getNodeReports();
  LOG.info("{}", nodeReports);

  for (NodeReport nr: nodeReports) {
    if (!nr.getNodeState().isUnusable()) {
      return;
    }
  }
  fail("Yarn Mini cluster should have at least one usable node.");
}
 
Example #20
Source File: TestDelegationTokenRenewer.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testDTRonAppSubmission()
    throws IOException, InterruptedException, BrokenBarrierException {
  final Credentials credsx = new Credentials();
  final Token<DelegationTokenIdentifier> tokenx = mock(Token.class);
  when(tokenx.getKind()).thenReturn(new Text("HDFS_DELEGATION_TOKEN"));
  DelegationTokenIdentifier dtId1 = 
      new DelegationTokenIdentifier(new Text("user1"), new Text("renewer"),
        new Text("user1"));
  when(tokenx.decodeIdentifier()).thenReturn(dtId1);
  credsx.addToken(new Text("token"), tokenx);
  doReturn(true).when(tokenx).isManaged();
  doThrow(new IOException("boom"))
      .when(tokenx).renew(any(Configuration.class));
    // fire up the renewer
  final DelegationTokenRenewer dtr =
       createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext = mock(RMContext.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  dtr.setRMContext(mockContext);
  when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr);
  dtr.init(conf);
  dtr.start();

  try {
    dtr.addApplicationSync(mock(ApplicationId.class), credsx, false, "user");
    fail("Catch IOException on app submission");
  } catch (IOException e){
    Assert.assertTrue(e.getMessage().contains(tokenx.toString()));
    Assert.assertTrue(e.getCause().toString().contains("boom"));
  }

}
 
Example #21
Source File: TestDelegationTokenRenewer.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Basic idea of the test:
 * 0. Setup token KEEP_ALIVE
 * 1. create tokens.
 * 2. register them for renewal - to be cancelled on app complete
 * 3. Complete app.
 * 4. Verify token is alive within the KEEP_ALIVE time
 * 5. Verify token has been cancelled after the KEEP_ALIVE_TIME
 * @throws IOException
 * @throws URISyntaxException
 */
@Test(timeout=60000)
public void testDTKeepAlive1 () throws Exception {
  Configuration lconf = new Configuration(conf);
  lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
  //Keep tokens alive for 6 seconds.
  lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
  //Try removing tokens every second.
  lconf.setLong(
      YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS,
      1000l);
  DelegationTokenRenewer localDtr =
      createNewDelegationTokenRenewer(lconf, counter);
  RMContext mockContext = mock(RMContext.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      localDtr);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  localDtr.setRMContext(mockContext);
  localDtr.init(lconf);
  localDtr.start();
  
  MyFS dfs = (MyFS)FileSystem.get(lconf);
  LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+lconf.hashCode());
  
  Credentials ts = new Credentials();
  // get the delegation tokens
  MyToken token1 = dfs.getDelegationToken("user1");

  String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
  ts.addToken(new Text(nn1), token1);

  // register the tokens for renewal
  ApplicationId applicationId_0 =  BuilderUtils.newApplicationId(0, 0);
  localDtr.addApplicationAsync(applicationId_0, ts, true, "user");
  waitForEventsToGetProcessed(localDtr);
  if (!eventQueue.isEmpty()){
    Event evt = eventQueue.take();
    if (evt instanceof RMAppEvent) {
      Assert.assertEquals(((RMAppEvent)evt).getType(), RMAppEventType.START);
    } else {
      fail("RMAppEvent.START was expected!!");
    }
  }
  
  localDtr.applicationFinished(applicationId_0);
  waitForEventsToGetProcessed(localDtr);

  //Token should still be around. Renewal should not fail.
  token1.renew(lconf);

  //Allow the keepalive time to run out
  Thread.sleep(10000l);

  //The token should have been cancelled at this point. Renewal will fail.
  try {
    token1.renew(lconf);
    fail("Renewal of cancelled token should have failed");
  } catch (InvalidToken ite) {}
}
 
Example #22
Source File: TestRMWebAppFairScheduler.java    From big-c with Apache License 2.0 4 votes vote down vote up
public static ClientRMService mockClientRMService(RMContext rmContext) {
  return mock(ClientRMService.class);
}
 
Example #23
Source File: MockRMContext.java    From incubator-myriad with Apache License 2.0 4 votes vote down vote up
@Override
public ClientRMService getClientRMService() {
  return clientRMService;
}
 
Example #24
Source File: MockRMContext.java    From incubator-myriad with Apache License 2.0 4 votes vote down vote up
@Override
public void setClientRMService(ClientRMService clientRMService) {
  this.clientRMService = clientRMService;
}
 
Example #25
Source File: TestDelegationTokenRenewer.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test(timeout=20000)                                                         
public void testConcurrentAddApplication()                                  
    throws IOException, InterruptedException, BrokenBarrierException {       
  final CyclicBarrier startBarrier = new CyclicBarrier(2);                   
  final CyclicBarrier endBarrier = new CyclicBarrier(2);                     
                                                                             
  // this token uses barriers to block during renew                          
  final Credentials creds1 = new Credentials();                              
  final Token<DelegationTokenIdentifier> token1 = mock(Token.class);    
  when(token1.getKind()).thenReturn(new Text("HDFS_DELEGATION_TOKEN"));
  DelegationTokenIdentifier dtId1 = 
      new DelegationTokenIdentifier(new Text("user1"), new Text("renewer"),
        new Text("user1"));
  when(token1.decodeIdentifier()).thenReturn(dtId1);
  creds1.addToken(new Text("token"), token1);                                
  doReturn(true).when(token1).isManaged();                                   
  doAnswer(new Answer<Long>() {                                              
    public Long answer(InvocationOnMock invocation)                          
        throws InterruptedException, BrokenBarrierException { 
      startBarrier.await();                                                  
      endBarrier.await();                                                    
      return Long.MAX_VALUE;                                                 
    }}).when(token1).renew(any(Configuration.class));                        
                                                                             
  // this dummy token fakes renewing                                         
  final Credentials creds2 = new Credentials();                              
  final Token<DelegationTokenIdentifier> token2 = mock(Token.class);           
  when(token2.getKind()).thenReturn(new Text("HDFS_DELEGATION_TOKEN"));
  when(token2.decodeIdentifier()).thenReturn(dtId1);
  creds2.addToken(new Text("token"), token2);                                
  doReturn(true).when(token2).isManaged();                                   
  doReturn(Long.MAX_VALUE).when(token2).renew(any(Configuration.class));     
                                                                             
  // fire up the renewer                                                     
  final DelegationTokenRenewer dtr =
      createNewDelegationTokenRenewer(conf, counter);           
  RMContext mockContext = mock(RMContext.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  ClientRMService mockClientRMService = mock(ClientRMService.class);         
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);    
  InetSocketAddress sockAddr =                                               
      InetSocketAddress.createUnresolved("localhost", 1234);                 
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);           
  dtr.setRMContext(mockContext);  
  when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr);
  dtr.init(conf);
  dtr.start();                                                                           
  // submit a job that blocks during renewal                                 
  Thread submitThread = new Thread() {                                       
    @Override                                                                
    public void run() {
      dtr.addApplicationAsync(mock(ApplicationId.class), creds1, false, "user");
    }                                                                        
  };                                                                         
  submitThread.start();                                                      
                                                                             
  // wait till 1st submit blocks, then submit another
  startBarrier.await();                           
  dtr.addApplicationAsync(mock(ApplicationId.class), creds2, false, "user");
  // signal 1st to complete                                                  
  endBarrier.await();                                                        
  submitThread.join(); 
}
 
Example #26
Source File: TestDelegationTokenRenewer.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Basic idea of the test:
 * 0. Setup token KEEP_ALIVE
 * 1. create tokens.
 * 2. register them for renewal - to be cancelled on app complete
 * 3. Complete app.
 * 4. Verify token is alive within the KEEP_ALIVE time
 * 5. Send an explicity KEEP_ALIVE_REQUEST
 * 6. Verify token KEEP_ALIVE time is renewed.
 * 7. Verify token has been cancelled after the renewed KEEP_ALIVE_TIME.
 * @throws IOException
 * @throws URISyntaxException
 */
@Test(timeout=60000)
public void testDTKeepAlive2() throws Exception {
  Configuration lconf = new Configuration(conf);
  lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
  //Keep tokens alive for 6 seconds.
  lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
  //Try removing tokens every second.
  lconf.setLong(
      YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS,
      1000l);
  DelegationTokenRenewer localDtr =
      createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext = mock(RMContext.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      localDtr);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  localDtr.setRMContext(mockContext);
  localDtr.init(lconf);
  localDtr.start();
  
  MyFS dfs = (MyFS)FileSystem.get(lconf);
  LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+lconf.hashCode());

  Credentials ts = new Credentials();
  // get the delegation tokens
  MyToken token1 = dfs.getDelegationToken("user1");
  
  String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
  ts.addToken(new Text(nn1), token1);

  // register the tokens for renewal
  ApplicationId applicationId_0 =  BuilderUtils.newApplicationId(0, 0);
  localDtr.addApplicationAsync(applicationId_0, ts, true, "user");
  localDtr.applicationFinished(applicationId_0);
  waitForEventsToGetProcessed(delegationTokenRenewer);
  //Send another keep alive.
  localDtr.updateKeepAliveApplications(Collections
      .singletonList(applicationId_0));
  //Renewal should not fail.
  token1.renew(lconf);
  //Token should be around after this. 
  Thread.sleep(4500l);
  //Renewal should not fail. - ~1.5 seconds for keepalive timeout.
  token1.renew(lconf);
  //Allow the keepalive time to run out
  Thread.sleep(3000l);
  //The token should have been cancelled at this point. Renewal will fail.
  try {
    token1.renew(lconf);
    fail("Renewal of cancelled token should have failed");
  } catch (InvalidToken ite) {}
}
 
Example #27
Source File: TestRMWebAppFairScheduler.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public static ClientRMService mockClientRMService(RMContext rmContext) {
  return mock(ClientRMService.class);
}
 
Example #28
Source File: TestDelegationTokenRenewer.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test(timeout=20000)                                                         
public void testConcurrentAddApplication()                                  
    throws IOException, InterruptedException, BrokenBarrierException {       
  final CyclicBarrier startBarrier = new CyclicBarrier(2);                   
  final CyclicBarrier endBarrier = new CyclicBarrier(2);                     
                                                                             
  // this token uses barriers to block during renew                          
  final Credentials creds1 = new Credentials();                              
  final Token<DelegationTokenIdentifier> token1 = mock(Token.class);    
  when(token1.getKind()).thenReturn(new Text("HDFS_DELEGATION_TOKEN"));
  DelegationTokenIdentifier dtId1 = 
      new DelegationTokenIdentifier(new Text("user1"), new Text("renewer"),
        new Text("user1"));
  when(token1.decodeIdentifier()).thenReturn(dtId1);
  creds1.addToken(new Text("token"), token1);                                
  doReturn(true).when(token1).isManaged();                                   
  doAnswer(new Answer<Long>() {                                              
    public Long answer(InvocationOnMock invocation)                          
        throws InterruptedException, BrokenBarrierException { 
      startBarrier.await();                                                  
      endBarrier.await();                                                    
      return Long.MAX_VALUE;                                                 
    }}).when(token1).renew(any(Configuration.class));                        
                                                                             
  // this dummy token fakes renewing                                         
  final Credentials creds2 = new Credentials();                              
  final Token<DelegationTokenIdentifier> token2 = mock(Token.class);           
  when(token2.getKind()).thenReturn(new Text("HDFS_DELEGATION_TOKEN"));
  when(token2.decodeIdentifier()).thenReturn(dtId1);
  creds2.addToken(new Text("token"), token2);                                
  doReturn(true).when(token2).isManaged();                                   
  doReturn(Long.MAX_VALUE).when(token2).renew(any(Configuration.class));     
                                                                             
  // fire up the renewer                                                     
  final DelegationTokenRenewer dtr =
      createNewDelegationTokenRenewer(conf, counter);           
  RMContext mockContext = mock(RMContext.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  ClientRMService mockClientRMService = mock(ClientRMService.class);         
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);    
  InetSocketAddress sockAddr =                                               
      InetSocketAddress.createUnresolved("localhost", 1234);                 
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);           
  dtr.setRMContext(mockContext);  
  when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr);
  dtr.init(conf);
  dtr.start();                                                                           
  // submit a job that blocks during renewal                                 
  Thread submitThread = new Thread() {                                       
    @Override                                                                
    public void run() {
      dtr.addApplicationAsync(mock(ApplicationId.class), creds1, false, "user");
    }                                                                        
  };                                                                         
  submitThread.start();                                                      
                                                                             
  // wait till 1st submit blocks, then submit another
  startBarrier.await();                           
  dtr.addApplicationAsync(mock(ApplicationId.class), creds2, false, "user");
  // signal 1st to complete                                                  
  endBarrier.await();                                                        
  submitThread.join(); 
}
 
Example #29
Source File: TestDelegationTokenRenewer.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Basic idea of the test:
 * 0. Setup token KEEP_ALIVE
 * 1. create tokens.
 * 2. register them for renewal - to be cancelled on app complete
 * 3. Complete app.
 * 4. Verify token is alive within the KEEP_ALIVE time
 * 5. Send an explicity KEEP_ALIVE_REQUEST
 * 6. Verify token KEEP_ALIVE time is renewed.
 * 7. Verify token has been cancelled after the renewed KEEP_ALIVE_TIME.
 * @throws IOException
 * @throws URISyntaxException
 */
@Test(timeout=60000)
public void testDTKeepAlive2() throws Exception {
  Configuration lconf = new Configuration(conf);
  lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
  //Keep tokens alive for 6 seconds.
  lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
  //Try removing tokens every second.
  lconf.setLong(
      YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS,
      1000l);
  DelegationTokenRenewer localDtr =
      createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext = mock(RMContext.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      localDtr);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  localDtr.setRMContext(mockContext);
  localDtr.init(lconf);
  localDtr.start();
  
  MyFS dfs = (MyFS)FileSystem.get(lconf);
  LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+lconf.hashCode());

  Credentials ts = new Credentials();
  // get the delegation tokens
  MyToken token1 = dfs.getDelegationToken("user1");
  
  String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
  ts.addToken(new Text(nn1), token1);

  // register the tokens for renewal
  ApplicationId applicationId_0 =  BuilderUtils.newApplicationId(0, 0);
  localDtr.addApplicationAsync(applicationId_0, ts, true, "user");
  localDtr.applicationFinished(applicationId_0);
  waitForEventsToGetProcessed(delegationTokenRenewer);
  //Send another keep alive.
  localDtr.updateKeepAliveApplications(Collections
      .singletonList(applicationId_0));
  //Renewal should not fail.
  token1.renew(lconf);
  //Token should be around after this. 
  Thread.sleep(4500l);
  //Renewal should not fail. - ~1.5 seconds for keepalive timeout.
  token1.renew(lconf);
  //Allow the keepalive time to run out
  Thread.sleep(3000l);
  //The token should have been cancelled at this point. Renewal will fail.
  try {
    token1.renew(lconf);
    fail("Renewal of cancelled token should have failed");
  } catch (InvalidToken ite) {}
}
 
Example #30
Source File: TestDelegationTokenRenewer.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Basic idea of the test:
 * 0. Setup token KEEP_ALIVE
 * 1. create tokens.
 * 2. register them for renewal - to be cancelled on app complete
 * 3. Complete app.
 * 4. Verify token is alive within the KEEP_ALIVE time
 * 5. Verify token has been cancelled after the KEEP_ALIVE_TIME
 * @throws IOException
 * @throws URISyntaxException
 */
@Test(timeout=60000)
public void testDTKeepAlive1 () throws Exception {
  Configuration lconf = new Configuration(conf);
  lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
  //Keep tokens alive for 6 seconds.
  lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
  //Try removing tokens every second.
  lconf.setLong(
      YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS,
      1000l);
  DelegationTokenRenewer localDtr =
      createNewDelegationTokenRenewer(lconf, counter);
  RMContext mockContext = mock(RMContext.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      localDtr);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  localDtr.setRMContext(mockContext);
  localDtr.init(lconf);
  localDtr.start();
  
  MyFS dfs = (MyFS)FileSystem.get(lconf);
  LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+lconf.hashCode());
  
  Credentials ts = new Credentials();
  // get the delegation tokens
  MyToken token1 = dfs.getDelegationToken("user1");

  String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
  ts.addToken(new Text(nn1), token1);

  // register the tokens for renewal
  ApplicationId applicationId_0 =  BuilderUtils.newApplicationId(0, 0);
  localDtr.addApplicationAsync(applicationId_0, ts, true, "user");
  waitForEventsToGetProcessed(localDtr);
  if (!eventQueue.isEmpty()){
    Event evt = eventQueue.take();
    if (evt instanceof RMAppEvent) {
      Assert.assertEquals(((RMAppEvent)evt).getType(), RMAppEventType.START);
    } else {
      fail("RMAppEvent.START was expected!!");
    }
  }
  
  localDtr.applicationFinished(applicationId_0);
  waitForEventsToGetProcessed(localDtr);

  //Token should still be around. Renewal should not fail.
  token1.renew(lconf);

  //Allow the keepalive time to run out
  Thread.sleep(10000l);

  //The token should have been cancelled at this point. Renewal will fail.
  try {
    token1.renew(lconf);
    fail("Renewal of cancelled token should have failed");
  } catch (InvalidToken ite) {}
}