Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.ClientRMService

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.ClientRMService. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestDelegationTokenRenewer.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  counter = new AtomicInteger(0);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  eventQueue = new LinkedBlockingQueue<Event>();
  dispatcher = new AsyncDispatcher(eventQueue);
  Renewer.reset();
  delegationTokenRenewer = createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext =  mock(RMContext.class);
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      delegationTokenRenewer);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  delegationTokenRenewer.setRMContext(mockContext);
  delegationTokenRenewer.init(conf);
  delegationTokenRenewer.start();
}
 
Example 2
Source Project: big-c   Source File: TestDelegationTokenRenewer.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  counter = new AtomicInteger(0);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  eventQueue = new LinkedBlockingQueue<Event>();
  dispatcher = new AsyncDispatcher(eventQueue);
  Renewer.reset();
  delegationTokenRenewer = createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext =  mock(RMContext.class);
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      delegationTokenRenewer);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  delegationTokenRenewer.setRMContext(mockContext);
  delegationTokenRenewer.init(conf);
  delegationTokenRenewer.start();
}
 
Example 3
Source Project: hadoop   Source File: TestDelegationTokenRenewer.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testDTRonAppSubmission()
    throws IOException, InterruptedException, BrokenBarrierException {
  final Credentials credsx = new Credentials();
  final Token<DelegationTokenIdentifier> tokenx = mock(Token.class);
  when(tokenx.getKind()).thenReturn(new Text("HDFS_DELEGATION_TOKEN"));
  DelegationTokenIdentifier dtId1 = 
      new DelegationTokenIdentifier(new Text("user1"), new Text("renewer"),
        new Text("user1"));
  when(tokenx.decodeIdentifier()).thenReturn(dtId1);
  credsx.addToken(new Text("token"), tokenx);
  doReturn(true).when(tokenx).isManaged();
  doThrow(new IOException("boom"))
      .when(tokenx).renew(any(Configuration.class));
    // fire up the renewer
  final DelegationTokenRenewer dtr =
       createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext = mock(RMContext.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  dtr.setRMContext(mockContext);
  when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr);
  dtr.init(conf);
  dtr.start();

  try {
    dtr.addApplicationSync(mock(ApplicationId.class), credsx, false, "user");
    fail("Catch IOException on app submission");
  } catch (IOException e){
    Assert.assertTrue(e.getMessage().contains(tokenx.toString()));
    Assert.assertTrue(e.getCause().toString().contains("boom"));
  }

}
 
Example 4
@Test
public void testStartupFailure() throws Exception {
  Configuration conf = new Configuration();
  DelegationTokenRenewer delegationTokenRenewer =
      new DelegationTokenRenewer();
  RMContext mockContext = mock(RMContext.class);
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  delegationTokenRenewer.setRMContext(mockContext);
  delegationTokenRenewer.init(conf);
  delegationTokenRenewer.stop();
}
 
Example 5
Source Project: hadoop   Source File: TestRMWebServices.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAppsRace() throws Exception {
  // mock up an RM that returns app reports for apps that don't exist
  // in the RMApps list
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationReport mockReport = mock(ApplicationReport.class);
  when(mockReport.getApplicationId()).thenReturn(appId);
  GetApplicationsResponse mockAppsResponse =
      mock(GetApplicationsResponse.class);
  when(mockAppsResponse.getApplicationList())
    .thenReturn(Arrays.asList(new ApplicationReport[] { mockReport }));
  ClientRMService mockClientSvc = mock(ClientRMService.class);
  when(mockClientSvc.getApplications(isA(GetApplicationsRequest.class),
      anyBoolean())).thenReturn(mockAppsResponse);
  ResourceManager mockRM = mock(ResourceManager.class);
  RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
      null, null, null, null, null);
  when(mockRM.getRMContext()).thenReturn(rmContext);
  when(mockRM.getClientRMService()).thenReturn(mockClientSvc);

  RMWebServices webSvc = new RMWebServices(mockRM, new Configuration(),
      mock(HttpServletResponse.class));

  final Set<String> emptySet =
      Collections.unmodifiableSet(Collections.<String>emptySet());

  // verify we don't get any apps when querying
  HttpServletRequest mockHsr = mock(HttpServletRequest.class);
  AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
      null, null, null, null, null, null, null, emptySet, emptySet);
  assertTrue(appsInfo.getApps().isEmpty());

  // verify we don't get an NPE when specifying a final status query
  appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED",
      null, null, null, null, null, null, null, emptySet, emptySet);
  assertTrue(appsInfo.getApps().isEmpty());
}
 
Example 6
Source Project: hadoop   Source File: TestRMWebApp.java    License: Apache License 2.0 5 votes vote down vote up
public static ResourceManager mockRm(RMContext rmContext) throws IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs = mockCapacityScheduler();
  ApplicationACLsManager aclMgr = mockAppACLsManager();
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getApplicationACLsManager()).thenReturn(aclMgr);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 
Example 7
Source Project: hadoop   Source File: TestRMWebApp.java    License: Apache License 2.0 5 votes vote down vote up
public static ClientRMService mockClientRMService(RMContext rmContext) {
  ClientRMService clientRMService = mock(ClientRMService.class);
  List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
  for (RMApp app : rmContext.getRMApps().values()) {
    ApplicationReport appReport =
        ApplicationReport.newInstance(
            app.getApplicationId(), (ApplicationAttemptId) null,
            app.getUser(), app.getQueue(),
            app.getName(), (String) null, 0, (Token) null,
            app.createApplicationState(),
            app.getDiagnostics().toString(), (String) null,
            app.getStartTime(), app.getFinishTime(),
            app.getFinalApplicationStatus(),
            (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
            app.getProgress(), app.getApplicationType(), (Token) null);
    appReports.add(appReport);
  }
  GetApplicationsResponse response = mock(GetApplicationsResponse.class);
  when(response.getApplicationList()).thenReturn(appReports);
  try {
    when(clientRMService.getApplications(any(GetApplicationsRequest.class)))
        .thenReturn(response);
  } catch (YarnException e) {
    Assert.fail("Exception is not expteced.");
  }
  return clientRMService;
}
 
Example 8
Source Project: hadoop   Source File: TestRMWebAppFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private static ResourceManager mockRm(RMContext rmContext) throws
    IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs = mockFairScheduler();
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 
Example 9
Source Project: hadoop   Source File: TestRMWebAppFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private static ResourceManager mockRmWithApps(RMContext rmContext) throws
    IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs =  mockFairSchedulerWithoutApps(rmContext);
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 
Example 10
Source Project: hadoop   Source File: ProtocolHATestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected ResourceManager createResourceManager() {
  return new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
      // Don't try to login using keytab in the testcases.
    }
    @Override
    protected ClientRMService createClientRMService() {
      if (overrideClientRMService) {
        return new CustomedClientRMService(this.rmContext, this.scheduler,
            this.rmAppManager, this.applicationACLsManager,
            this.queueACLsManager,
            this.rmContext.getRMDelegationTokenSecretManager());
      }
      return super.createClientRMService();
    }
    @Override
    protected ResourceTrackerService createResourceTrackerService() {
      if (overrideRTS) {
        return new CustomedResourceTrackerService(this.rmContext,
            this.nodesListManager, this.nmLivelinessMonitor,
            this.rmContext.getContainerTokenSecretManager(),
            this.rmContext.getNMTokenSecretManager());
      }
      return super.createResourceTrackerService();
    }
    @Override
    protected ApplicationMasterService createApplicationMasterService() {
      if (overrideApplicationMasterService) {
        return new CustomedApplicationMasterService(this.rmContext,
            this.scheduler);
      }
      return super.createApplicationMasterService();
    }
  };
}
 
Example 11
Source Project: twill   Source File: TwillTester.java    License: Apache License 2.0 5 votes vote down vote up
public ApplicationResourceUsageReport getApplicationResourceReport(String appId) throws Exception {
  List<String> splits = Lists.newArrayList(Splitter.on('_').split(appId));
  Preconditions.checkArgument(splits.size() == 3, "Invalid application id - " + appId);
  ApplicationId applicationId = ApplicationId.newInstance(Long.parseLong(splits.get(1)),
                                                          Integer.parseInt(splits.get(2)));

  ClientRMService clientRMService = cluster.getResourceManager().getClientRMService();
  GetApplicationReportRequest request = Records.newRecord(GetApplicationReportRequest.class);
  request.setApplicationId(applicationId);
  return clientRMService.getApplicationReport(request)
    .getApplicationReport().getApplicationResourceUsageReport();
}
 
Example 12
Source Project: big-c   Source File: TestDelegationTokenRenewer.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout=20000)
public void testDTRonAppSubmission()
    throws IOException, InterruptedException, BrokenBarrierException {
  final Credentials credsx = new Credentials();
  final Token<DelegationTokenIdentifier> tokenx = mock(Token.class);
  when(tokenx.getKind()).thenReturn(new Text("HDFS_DELEGATION_TOKEN"));
  DelegationTokenIdentifier dtId1 = 
      new DelegationTokenIdentifier(new Text("user1"), new Text("renewer"),
        new Text("user1"));
  when(tokenx.decodeIdentifier()).thenReturn(dtId1);
  credsx.addToken(new Text("token"), tokenx);
  doReturn(true).when(tokenx).isManaged();
  doThrow(new IOException("boom"))
      .when(tokenx).renew(any(Configuration.class));
    // fire up the renewer
  final DelegationTokenRenewer dtr =
       createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext = mock(RMContext.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  dtr.setRMContext(mockContext);
  when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr);
  dtr.init(conf);
  dtr.start();

  try {
    dtr.addApplicationSync(mock(ApplicationId.class), credsx, false, "user");
    fail("Catch IOException on app submission");
  } catch (IOException e){
    Assert.assertTrue(e.getMessage().contains(tokenx.toString()));
    Assert.assertTrue(e.getCause().toString().contains("boom"));
  }

}
 
Example 13
Source Project: big-c   Source File: TestDelegationTokenRenewerLifecycle.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testStartupFailure() throws Exception {
  Configuration conf = new Configuration();
  DelegationTokenRenewer delegationTokenRenewer =
      new DelegationTokenRenewer();
  RMContext mockContext = mock(RMContext.class);
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  delegationTokenRenewer.setRMContext(mockContext);
  delegationTokenRenewer.init(conf);
  delegationTokenRenewer.stop();
}
 
Example 14
Source Project: big-c   Source File: TestRMWebServices.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAppsRace() throws Exception {
  // mock up an RM that returns app reports for apps that don't exist
  // in the RMApps list
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationReport mockReport = mock(ApplicationReport.class);
  when(mockReport.getApplicationId()).thenReturn(appId);
  GetApplicationsResponse mockAppsResponse =
      mock(GetApplicationsResponse.class);
  when(mockAppsResponse.getApplicationList())
    .thenReturn(Arrays.asList(new ApplicationReport[] { mockReport }));
  ClientRMService mockClientSvc = mock(ClientRMService.class);
  when(mockClientSvc.getApplications(isA(GetApplicationsRequest.class),
      anyBoolean())).thenReturn(mockAppsResponse);
  ResourceManager mockRM = mock(ResourceManager.class);
  RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
      null, null, null, null, null);
  when(mockRM.getRMContext()).thenReturn(rmContext);
  when(mockRM.getClientRMService()).thenReturn(mockClientSvc);

  RMWebServices webSvc = new RMWebServices(mockRM, new Configuration(),
      mock(HttpServletResponse.class));

  final Set<String> emptySet =
      Collections.unmodifiableSet(Collections.<String>emptySet());

  // verify we don't get any apps when querying
  HttpServletRequest mockHsr = mock(HttpServletRequest.class);
  AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
      null, null, null, null, null, null, null, emptySet, emptySet);
  assertTrue(appsInfo.getApps().isEmpty());

  // verify we don't get an NPE when specifying a final status query
  appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED",
      null, null, null, null, null, null, null, emptySet, emptySet);
  assertTrue(appsInfo.getApps().isEmpty());
}
 
Example 15
Source Project: big-c   Source File: TestRMWebApp.java    License: Apache License 2.0 5 votes vote down vote up
public static ResourceManager mockRm(RMContext rmContext) throws IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs = mockCapacityScheduler();
  ApplicationACLsManager aclMgr = mockAppACLsManager();
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getApplicationACLsManager()).thenReturn(aclMgr);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 
Example 16
Source Project: big-c   Source File: TestRMWebApp.java    License: Apache License 2.0 5 votes vote down vote up
public static ClientRMService mockClientRMService(RMContext rmContext) {
  ClientRMService clientRMService = mock(ClientRMService.class);
  List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
  for (RMApp app : rmContext.getRMApps().values()) {
    ApplicationReport appReport =
        ApplicationReport.newInstance(
            app.getApplicationId(), (ApplicationAttemptId) null,
            app.getUser(), app.getQueue(),
            app.getName(), (String) null, 0, (Token) null,
            app.createApplicationState(),
            app.getDiagnostics().toString(), (String) null,
            app.getStartTime(), app.getFinishTime(),
            app.getFinalApplicationStatus(),
            (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
            app.getProgress(), app.getApplicationType(), (Token) null);
    appReports.add(appReport);
  }
  GetApplicationsResponse response = mock(GetApplicationsResponse.class);
  when(response.getApplicationList()).thenReturn(appReports);
  try {
    when(clientRMService.getApplications(any(GetApplicationsRequest.class)))
        .thenReturn(response);
  } catch (YarnException e) {
    Assert.fail("Exception is not expteced.");
  }
  return clientRMService;
}
 
Example 17
Source Project: big-c   Source File: TestRMWebAppFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private static ResourceManager mockRm(RMContext rmContext) throws
    IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs = mockFairScheduler();
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 
Example 18
Source Project: big-c   Source File: TestRMWebAppFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private static ResourceManager mockRmWithApps(RMContext rmContext) throws
    IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs =  mockFairSchedulerWithoutApps(rmContext);
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 
Example 19
Source Project: big-c   Source File: ProtocolHATestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected ResourceManager createResourceManager() {
  return new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
      // Don't try to login using keytab in the testcases.
    }
    @Override
    protected ClientRMService createClientRMService() {
      if (overrideClientRMService) {
        return new CustomedClientRMService(this.rmContext, this.scheduler,
            this.rmAppManager, this.applicationACLsManager,
            this.queueACLsManager,
            this.rmContext.getRMDelegationTokenSecretManager());
      }
      return super.createClientRMService();
    }
    @Override
    protected ResourceTrackerService createResourceTrackerService() {
      if (overrideRTS) {
        return new CustomedResourceTrackerService(this.rmContext,
            this.nodesListManager, this.nmLivelinessMonitor,
            this.rmContext.getContainerTokenSecretManager(),
            this.rmContext.getNMTokenSecretManager());
      }
      return super.createResourceTrackerService();
    }
    @Override
    protected ApplicationMasterService createApplicationMasterService() {
      if (overrideApplicationMasterService) {
        return new CustomedApplicationMasterService(this.rmContext,
            this.scheduler);
      }
      return super.createApplicationMasterService();
    }
  };
}
 
Example 20
Source Project: attic-apex-core   Source File: StramMiniClusterTest.java    License: Apache License 2.0 5 votes vote down vote up
private void checkNodeState() throws YarnException
{
  GetClusterNodesRequest request = Records.newRecord(GetClusterNodesRequest.class);
  ClientRMService clientRMService = yarnCluster.getResourceManager().getClientRMService();
  GetClusterNodesResponse response = clientRMService.getClusterNodes(request);
  List<NodeReport> nodeReports = response.getNodeReports();
  LOG.info("{}", nodeReports);

  for (NodeReport nr: nodeReports) {
    if (!nr.getNodeState().isUnusable()) {
      return;
    }
  }
  fail("Yarn Mini cluster should have at least one usable node.");
}
 
Example 21
Source Project: hadoop   Source File: TestDelegationTokenRenewer.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Basic idea of the test:
 * 0. Setup token KEEP_ALIVE
 * 1. create tokens.
 * 2. register them for renewal - to be cancelled on app complete
 * 3. Complete app.
 * 4. Verify token is alive within the KEEP_ALIVE time
 * 5. Verify token has been cancelled after the KEEP_ALIVE_TIME
 * @throws IOException
 * @throws URISyntaxException
 */
@Test(timeout=60000)
public void testDTKeepAlive1 () throws Exception {
  Configuration lconf = new Configuration(conf);
  lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
  //Keep tokens alive for 6 seconds.
  lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
  //Try removing tokens every second.
  lconf.setLong(
      YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS,
      1000l);
  DelegationTokenRenewer localDtr =
      createNewDelegationTokenRenewer(lconf, counter);
  RMContext mockContext = mock(RMContext.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      localDtr);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  localDtr.setRMContext(mockContext);
  localDtr.init(lconf);
  localDtr.start();
  
  MyFS dfs = (MyFS)FileSystem.get(lconf);
  LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+lconf.hashCode());
  
  Credentials ts = new Credentials();
  // get the delegation tokens
  MyToken token1 = dfs.getDelegationToken("user1");

  String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
  ts.addToken(new Text(nn1), token1);

  // register the tokens for renewal
  ApplicationId applicationId_0 =  BuilderUtils.newApplicationId(0, 0);
  localDtr.addApplicationAsync(applicationId_0, ts, true, "user");
  waitForEventsToGetProcessed(localDtr);
  if (!eventQueue.isEmpty()){
    Event evt = eventQueue.take();
    if (evt instanceof RMAppEvent) {
      Assert.assertEquals(((RMAppEvent)evt).getType(), RMAppEventType.START);
    } else {
      fail("RMAppEvent.START was expected!!");
    }
  }
  
  localDtr.applicationFinished(applicationId_0);
  waitForEventsToGetProcessed(localDtr);

  //Token should still be around. Renewal should not fail.
  token1.renew(lconf);

  //Allow the keepalive time to run out
  Thread.sleep(10000l);

  //The token should have been cancelled at this point. Renewal will fail.
  try {
    token1.renew(lconf);
    fail("Renewal of cancelled token should have failed");
  } catch (InvalidToken ite) {}
}
 
Example 22
Source Project: hadoop   Source File: TestDelegationTokenRenewer.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Basic idea of the test:
 * 0. Setup token KEEP_ALIVE
 * 1. create tokens.
 * 2. register them for renewal - to be cancelled on app complete
 * 3. Complete app.
 * 4. Verify token is alive within the KEEP_ALIVE time
 * 5. Send an explicity KEEP_ALIVE_REQUEST
 * 6. Verify token KEEP_ALIVE time is renewed.
 * 7. Verify token has been cancelled after the renewed KEEP_ALIVE_TIME.
 * @throws IOException
 * @throws URISyntaxException
 */
@Test(timeout=60000)
public void testDTKeepAlive2() throws Exception {
  Configuration lconf = new Configuration(conf);
  lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
  //Keep tokens alive for 6 seconds.
  lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
  //Try removing tokens every second.
  lconf.setLong(
      YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS,
      1000l);
  DelegationTokenRenewer localDtr =
      createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext = mock(RMContext.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      localDtr);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  localDtr.setRMContext(mockContext);
  localDtr.init(lconf);
  localDtr.start();
  
  MyFS dfs = (MyFS)FileSystem.get(lconf);
  LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+lconf.hashCode());

  Credentials ts = new Credentials();
  // get the delegation tokens
  MyToken token1 = dfs.getDelegationToken("user1");
  
  String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
  ts.addToken(new Text(nn1), token1);

  // register the tokens for renewal
  ApplicationId applicationId_0 =  BuilderUtils.newApplicationId(0, 0);
  localDtr.addApplicationAsync(applicationId_0, ts, true, "user");
  localDtr.applicationFinished(applicationId_0);
  waitForEventsToGetProcessed(delegationTokenRenewer);
  //Send another keep alive.
  localDtr.updateKeepAliveApplications(Collections
      .singletonList(applicationId_0));
  //Renewal should not fail.
  token1.renew(lconf);
  //Token should be around after this. 
  Thread.sleep(4500l);
  //Renewal should not fail. - ~1.5 seconds for keepalive timeout.
  token1.renew(lconf);
  //Allow the keepalive time to run out
  Thread.sleep(3000l);
  //The token should have been cancelled at this point. Renewal will fail.
  try {
    token1.renew(lconf);
    fail("Renewal of cancelled token should have failed");
  } catch (InvalidToken ite) {}
}
 
Example 23
Source Project: hadoop   Source File: TestDelegationTokenRenewer.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout=20000)                                                         
public void testConcurrentAddApplication()                                  
    throws IOException, InterruptedException, BrokenBarrierException {       
  final CyclicBarrier startBarrier = new CyclicBarrier(2);                   
  final CyclicBarrier endBarrier = new CyclicBarrier(2);                     
                                                                             
  // this token uses barriers to block during renew                          
  final Credentials creds1 = new Credentials();                              
  final Token<DelegationTokenIdentifier> token1 = mock(Token.class);    
  when(token1.getKind()).thenReturn(new Text("HDFS_DELEGATION_TOKEN"));
  DelegationTokenIdentifier dtId1 = 
      new DelegationTokenIdentifier(new Text("user1"), new Text("renewer"),
        new Text("user1"));
  when(token1.decodeIdentifier()).thenReturn(dtId1);
  creds1.addToken(new Text("token"), token1);                                
  doReturn(true).when(token1).isManaged();                                   
  doAnswer(new Answer<Long>() {                                              
    public Long answer(InvocationOnMock invocation)                          
        throws InterruptedException, BrokenBarrierException { 
      startBarrier.await();                                                  
      endBarrier.await();                                                    
      return Long.MAX_VALUE;                                                 
    }}).when(token1).renew(any(Configuration.class));                        
                                                                             
  // this dummy token fakes renewing                                         
  final Credentials creds2 = new Credentials();                              
  final Token<DelegationTokenIdentifier> token2 = mock(Token.class);           
  when(token2.getKind()).thenReturn(new Text("HDFS_DELEGATION_TOKEN"));
  when(token2.decodeIdentifier()).thenReturn(dtId1);
  creds2.addToken(new Text("token"), token2);                                
  doReturn(true).when(token2).isManaged();                                   
  doReturn(Long.MAX_VALUE).when(token2).renew(any(Configuration.class));     
                                                                             
  // fire up the renewer                                                     
  final DelegationTokenRenewer dtr =
      createNewDelegationTokenRenewer(conf, counter);           
  RMContext mockContext = mock(RMContext.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  ClientRMService mockClientRMService = mock(ClientRMService.class);         
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);    
  InetSocketAddress sockAddr =                                               
      InetSocketAddress.createUnresolved("localhost", 1234);                 
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);           
  dtr.setRMContext(mockContext);  
  when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr);
  dtr.init(conf);
  dtr.start();                                                                           
  // submit a job that blocks during renewal                                 
  Thread submitThread = new Thread() {                                       
    @Override                                                                
    public void run() {
      dtr.addApplicationAsync(mock(ApplicationId.class), creds1, false, "user");
    }                                                                        
  };                                                                         
  submitThread.start();                                                      
                                                                             
  // wait till 1st submit blocks, then submit another
  startBarrier.await();                           
  dtr.addApplicationAsync(mock(ApplicationId.class), creds2, false, "user");
  // signal 1st to complete                                                  
  endBarrier.await();                                                        
  submitThread.join(); 
}
 
Example 24
Source Project: hadoop   Source File: TestRMWebAppFairScheduler.java    License: Apache License 2.0 4 votes vote down vote up
public static ClientRMService mockClientRMService(RMContext rmContext) {
  return mock(ClientRMService.class);
}
 
Example 25
Source Project: big-c   Source File: TestDelegationTokenRenewer.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Basic idea of the test:
 * 0. Setup token KEEP_ALIVE
 * 1. create tokens.
 * 2. register them for renewal - to be cancelled on app complete
 * 3. Complete app.
 * 4. Verify token is alive within the KEEP_ALIVE time
 * 5. Verify token has been cancelled after the KEEP_ALIVE_TIME
 * @throws IOException
 * @throws URISyntaxException
 */
@Test(timeout=60000)
public void testDTKeepAlive1 () throws Exception {
  Configuration lconf = new Configuration(conf);
  lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
  //Keep tokens alive for 6 seconds.
  lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
  //Try removing tokens every second.
  lconf.setLong(
      YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS,
      1000l);
  DelegationTokenRenewer localDtr =
      createNewDelegationTokenRenewer(lconf, counter);
  RMContext mockContext = mock(RMContext.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      localDtr);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  localDtr.setRMContext(mockContext);
  localDtr.init(lconf);
  localDtr.start();
  
  MyFS dfs = (MyFS)FileSystem.get(lconf);
  LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+lconf.hashCode());
  
  Credentials ts = new Credentials();
  // get the delegation tokens
  MyToken token1 = dfs.getDelegationToken("user1");

  String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
  ts.addToken(new Text(nn1), token1);

  // register the tokens for renewal
  ApplicationId applicationId_0 =  BuilderUtils.newApplicationId(0, 0);
  localDtr.addApplicationAsync(applicationId_0, ts, true, "user");
  waitForEventsToGetProcessed(localDtr);
  if (!eventQueue.isEmpty()){
    Event evt = eventQueue.take();
    if (evt instanceof RMAppEvent) {
      Assert.assertEquals(((RMAppEvent)evt).getType(), RMAppEventType.START);
    } else {
      fail("RMAppEvent.START was expected!!");
    }
  }
  
  localDtr.applicationFinished(applicationId_0);
  waitForEventsToGetProcessed(localDtr);

  //Token should still be around. Renewal should not fail.
  token1.renew(lconf);

  //Allow the keepalive time to run out
  Thread.sleep(10000l);

  //The token should have been cancelled at this point. Renewal will fail.
  try {
    token1.renew(lconf);
    fail("Renewal of cancelled token should have failed");
  } catch (InvalidToken ite) {}
}
 
Example 26
Source Project: big-c   Source File: TestDelegationTokenRenewer.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Basic idea of the test:
 * 0. Setup token KEEP_ALIVE
 * 1. create tokens.
 * 2. register them for renewal - to be cancelled on app complete
 * 3. Complete app.
 * 4. Verify token is alive within the KEEP_ALIVE time
 * 5. Send an explicity KEEP_ALIVE_REQUEST
 * 6. Verify token KEEP_ALIVE time is renewed.
 * 7. Verify token has been cancelled after the renewed KEEP_ALIVE_TIME.
 * @throws IOException
 * @throws URISyntaxException
 */
@Test(timeout=60000)
public void testDTKeepAlive2() throws Exception {
  Configuration lconf = new Configuration(conf);
  lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
  //Keep tokens alive for 6 seconds.
  lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
  //Try removing tokens every second.
  lconf.setLong(
      YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS,
      1000l);
  DelegationTokenRenewer localDtr =
      createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext = mock(RMContext.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      localDtr);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  localDtr.setRMContext(mockContext);
  localDtr.init(lconf);
  localDtr.start();
  
  MyFS dfs = (MyFS)FileSystem.get(lconf);
  LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+lconf.hashCode());

  Credentials ts = new Credentials();
  // get the delegation tokens
  MyToken token1 = dfs.getDelegationToken("user1");
  
  String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
  ts.addToken(new Text(nn1), token1);

  // register the tokens for renewal
  ApplicationId applicationId_0 =  BuilderUtils.newApplicationId(0, 0);
  localDtr.addApplicationAsync(applicationId_0, ts, true, "user");
  localDtr.applicationFinished(applicationId_0);
  waitForEventsToGetProcessed(delegationTokenRenewer);
  //Send another keep alive.
  localDtr.updateKeepAliveApplications(Collections
      .singletonList(applicationId_0));
  //Renewal should not fail.
  token1.renew(lconf);
  //Token should be around after this. 
  Thread.sleep(4500l);
  //Renewal should not fail. - ~1.5 seconds for keepalive timeout.
  token1.renew(lconf);
  //Allow the keepalive time to run out
  Thread.sleep(3000l);
  //The token should have been cancelled at this point. Renewal will fail.
  try {
    token1.renew(lconf);
    fail("Renewal of cancelled token should have failed");
  } catch (InvalidToken ite) {}
}
 
Example 27
Source Project: big-c   Source File: TestDelegationTokenRenewer.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout=20000)                                                         
public void testConcurrentAddApplication()                                  
    throws IOException, InterruptedException, BrokenBarrierException {       
  final CyclicBarrier startBarrier = new CyclicBarrier(2);                   
  final CyclicBarrier endBarrier = new CyclicBarrier(2);                     
                                                                             
  // this token uses barriers to block during renew                          
  final Credentials creds1 = new Credentials();                              
  final Token<DelegationTokenIdentifier> token1 = mock(Token.class);    
  when(token1.getKind()).thenReturn(new Text("HDFS_DELEGATION_TOKEN"));
  DelegationTokenIdentifier dtId1 = 
      new DelegationTokenIdentifier(new Text("user1"), new Text("renewer"),
        new Text("user1"));
  when(token1.decodeIdentifier()).thenReturn(dtId1);
  creds1.addToken(new Text("token"), token1);                                
  doReturn(true).when(token1).isManaged();                                   
  doAnswer(new Answer<Long>() {                                              
    public Long answer(InvocationOnMock invocation)                          
        throws InterruptedException, BrokenBarrierException { 
      startBarrier.await();                                                  
      endBarrier.await();                                                    
      return Long.MAX_VALUE;                                                 
    }}).when(token1).renew(any(Configuration.class));                        
                                                                             
  // this dummy token fakes renewing                                         
  final Credentials creds2 = new Credentials();                              
  final Token<DelegationTokenIdentifier> token2 = mock(Token.class);           
  when(token2.getKind()).thenReturn(new Text("HDFS_DELEGATION_TOKEN"));
  when(token2.decodeIdentifier()).thenReturn(dtId1);
  creds2.addToken(new Text("token"), token2);                                
  doReturn(true).when(token2).isManaged();                                   
  doReturn(Long.MAX_VALUE).when(token2).renew(any(Configuration.class));     
                                                                             
  // fire up the renewer                                                     
  final DelegationTokenRenewer dtr =
      createNewDelegationTokenRenewer(conf, counter);           
  RMContext mockContext = mock(RMContext.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  ClientRMService mockClientRMService = mock(ClientRMService.class);         
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);    
  InetSocketAddress sockAddr =                                               
      InetSocketAddress.createUnresolved("localhost", 1234);                 
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);           
  dtr.setRMContext(mockContext);  
  when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr);
  dtr.init(conf);
  dtr.start();                                                                           
  // submit a job that blocks during renewal                                 
  Thread submitThread = new Thread() {                                       
    @Override                                                                
    public void run() {
      dtr.addApplicationAsync(mock(ApplicationId.class), creds1, false, "user");
    }                                                                        
  };                                                                         
  submitThread.start();                                                      
                                                                             
  // wait till 1st submit blocks, then submit another
  startBarrier.await();                           
  dtr.addApplicationAsync(mock(ApplicationId.class), creds2, false, "user");
  // signal 1st to complete                                                  
  endBarrier.await();                                                        
  submitThread.join(); 
}
 
Example 28
Source Project: big-c   Source File: TestRMWebAppFairScheduler.java    License: Apache License 2.0 4 votes vote down vote up
public static ClientRMService mockClientRMService(RMContext rmContext) {
  return mock(ClientRMService.class);
}
 
Example 29
Source Project: incubator-myriad   Source File: MockRMContext.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ClientRMService getClientRMService() {
  return clientRMService;
}
 
Example 30
Source Project: incubator-myriad   Source File: MockRMContext.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void setClientRMService(ClientRMService clientRMService) {
  this.clientRMService = clientRMService;
}