Java Code Examples for org.apache.hadoop.test.GenericTestUtils.LogCapturer#captureLogs()

@Test
public void testRemoveVolume() throws Exception {

  assertEquals(2, volumeSet.getVolumesList().size());

  // Remove a volume from VolumeSet
  volumeSet.removeVolume(volume1);
  assertEquals(1, volumeSet.getVolumesList().size());

  // Attempting to remove a volume which does not exist in VolumeSet should
  // log a warning.
  LogCapturer logs = LogCapturer.captureLogs(
      LogFactory.getLog(MutableVolumeSet.class));
  volumeSet.removeVolume(volume1);
  assertEquals(1, volumeSet.getVolumesList().size());
  String expectedLogMessage = "Volume : " +
      HddsVolumeUtil.getHddsRoot(volume1) + " does not exist in VolumeSet";
  assertTrue("Log output does not contain expected log message: "
      + expectedLogMessage, logs.getOutput().contains(expectedLogMessage));
}
 

/**
 * Tests the secure om Initialization success.
 */
@Test
public void testSecureOmInitializationSuccess() throws Exception {
  initSCM();
  // Create a secure SCM instance as om client will connect to it
  scm = StorageContainerManager.createSCM(conf);
  LogCapturer logs = LogCapturer.captureLogs(OzoneManager.getLogger());
  GenericTestUtils.setLogLevel(OzoneManager.getLogger(), INFO);

  setupOm(conf);
  try {
    om.start();
  } catch (Exception ex) {
    // Expects timeout failure from scmClient in om but om user login via
    // kerberos should succeed.
    assertTrue(logs.getOutput().contains("Ozone Manager login successful"));
  }
}
 

/**
 * Test functionality to get SCM signed certificate for OM.
 */
@Test
public void testSecureOmInitSuccess() throws Exception {
  LogCapturer omLogs =
      LogCapturer.captureLogs(OzoneManager.getLogger());
  omLogs.clearOutput();
  initSCM();
  try {
    scm = HddsTestUtils.getScm(conf);
    scm.start();

    OMStorage omStore = new OMStorage(conf);
    initializeOmStorage(omStore);
    OzoneManager.setTestSecureOmFlag(true);
    om = OzoneManager.createOm(conf);

    assertNotNull(om.getCertificateClient());
    assertNotNull(om.getCertificateClient().getPublicKey());
    assertNotNull(om.getCertificateClient().getPrivateKey());
    assertNotNull(om.getCertificateClient().getCertificate());
    assertTrue(omLogs.getOutput().contains("Init response: GETCERT"));
    assertTrue(omLogs.getOutput().contains("Successfully stored " +
        "SCM signed certificate"));
    X509Certificate certificate = om.getCertificateClient().getCertificate();
    validateCertificate(certificate);
    String pemEncodedCACert =
        scm.getSecurityProtocolServer().getCACertificate();
    X509Certificate caCert = CertificateCodec.getX509Cert(pemEncodedCACert);
    X509Certificate caCertStored = om.getCertificateClient()
        .getCertificate(caCert.getSerialNumber().toString());
    assertEquals(caCert, caCertStored);
  } finally {
    if (scm != null) {
      scm.stop();
    }
    IOUtils.closeQuietly(om);
  }

}
 

private void waitForDatanodeBlockDeletionStart()
    throws TimeoutException, InterruptedException {
  LogCapturer logCapturer =
      LogCapturer.captureLogs(DeleteBlocksCommandHandler.LOG);
  logCapturer.clearOutput();
  GenericTestUtils.waitFor(() -> logCapturer.getOutput()
          .contains("Start to delete container block"),
      500, 10000);
  Thread.sleep(1000);
}
 

/**
 * Waits for datanode command to be retried when datanode is dead.
 */
private void waitForDatanodeCommandRetry()
    throws TimeoutException, InterruptedException {
  cluster.shutdownHddsDatanode(0);
  LogCapturer logCapturer =
      LogCapturer.captureLogs(RetriableDatanodeEventWatcher.LOG);
  logCapturer.clearOutput();
  GenericTestUtils.waitFor(() -> logCapturer.getOutput()
          .contains("RetriableDatanodeCommand type=deleteBlocksCommand"),
      500, 5000);
  cluster.restartHddsDatanode(0, true);
}
 

@Test
public void testBlockDeletionTimeout() throws Exception {
  OzoneConfiguration conf = new OzoneConfiguration();
  conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10);
  conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 2);
  ContainerSet containerSet = new ContainerSet();
  createToDeleteBlocks(containerSet, conf, 1, 3, 1);

  // set timeout value as 1ns to trigger timeout behavior
  long timeout  = 1;
  OzoneContainer ozoneContainer = mockDependencies(containerSet);
  BlockDeletingService svc = new BlockDeletingService(ozoneContainer,
      TimeUnit.MILLISECONDS.toNanos(1000), timeout, TimeUnit.NANOSECONDS,
      conf);
  svc.start();

  LogCapturer log = LogCapturer.captureLogs(BackgroundService.LOG);
  GenericTestUtils.waitFor(() -> {
    if(log.getOutput().contains(
        "Background task executes timed out, retrying in next interval")) {
      log.stopCapturing();
      return true;
    }

    return false;
  }, 100, 1000);

  log.stopCapturing();
  svc.shutdown();

  // test for normal case that doesn't have timeout limitation
  timeout  = 0;
  createToDeleteBlocks(containerSet, conf, 1, 3, 1);
  svc = new BlockDeletingService(ozoneContainer,
      TimeUnit.MILLISECONDS.toNanos(1000), timeout, TimeUnit.MILLISECONDS,
      conf);
  svc.start();

  // get container meta data
  KeyValueContainer container =
      (KeyValueContainer) containerSet.getContainerIterator().next();
  KeyValueContainerData data = container.getContainerData();
  try (ReferenceCountedDB meta = BlockUtils.getDB(data, conf)) {

    LogCapturer newLog = LogCapturer.captureLogs(BackgroundService.LOG);
    GenericTestUtils.waitFor(() -> {
      try {
        return getUnderDeletionBlocksCount(meta) == 0;
      } catch (IOException ignored) {
      }
      return false;
    }, 100, 1000);
    newLog.stopCapturing();

    // The block deleting successfully and shouldn't catch timed
    // out warning log.
    Assert.assertFalse(newLog.getOutput().contains(
        "Background task executes timed out, retrying in next interval"));
  }
  svc.shutdown();
}
 

@Before
public void setUp() {
  capturer = LogCapturer.captureLogs(
      LoggerFactory.getLogger(SpaceUsageCheckFactory.class));
}
 

/**
 * Tests delegation token renewal.
 */
@Test
public void testDelegationTokenRenewal() throws Exception {
  GenericTestUtils
      .setLogLevel(LoggerFactory.getLogger(Server.class.getName()), INFO);
  LogCapturer omLogs = LogCapturer.captureLogs(OzoneManager.getLogger());

  // Setup secure OM for start.
  OzoneConfiguration newConf = new OzoneConfiguration(conf);
  int tokenMaxLifetime = 1000;
  newConf.setLong(DELEGATION_TOKEN_MAX_LIFETIME_KEY, tokenMaxLifetime);
  setupOm(newConf);
  long omVersion =
      RPC.getProtocolVersion(OzoneManagerProtocolPB.class);
  OzoneManager.setTestSecureOmFlag(true);
  // Start OM

  try {
    om.setCertClient(new CertificateClientTestImpl(conf));
    om.start();

    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();

    // Get first OM client which will authenticate via Kerberos
    omClient = new OzoneManagerProtocolClientSideTranslatorPB(
        OmTransportFactory.create(conf, ugi, null),
        RandomStringUtils.randomAscii(5));

    // Since client is already connected get a delegation token
    Token<OzoneTokenIdentifier> token = omClient.getDelegationToken(
        new Text("om"));

    // Check if token is of right kind and renewer is running om instance
    assertNotNull(token);
    assertEquals("OzoneToken", token.getKind().toString());
    assertEquals(OmUtils.getOmRpcAddress(conf),
        token.getService().toString());

    // Renew delegation token
    long expiryTime = omClient.renewDelegationToken(token);
    assertTrue(expiryTime > 0);
    omLogs.clearOutput();

    // Test failure of delegation renewal
    // 1. When token maxExpiryTime exceeds
    Thread.sleep(tokenMaxLifetime);
    OMException ex = LambdaTestUtils.intercept(OMException.class,
        "TOKEN_EXPIRED",
        () -> omClient.renewDelegationToken(token));
    assertEquals(TOKEN_EXPIRED, ex.getResult());
    omLogs.clearOutput();

    // 2. When renewer doesn't match (implicitly covers when renewer is
    // null or empty )
    Token<OzoneTokenIdentifier> token2 = omClient.getDelegationToken(
        new Text("randomService"));
    assertNotNull(token2);
    LambdaTestUtils.intercept(OMException.class,
        "Delegation token renewal failed",
        () -> omClient.renewDelegationToken(token2));
    assertTrue(omLogs.getOutput().contains(" with non-matching " +
        "renewer randomService"));
    omLogs.clearOutput();

    // 3. Test tampered token
    OzoneTokenIdentifier tokenId = OzoneTokenIdentifier.readProtoBuf(
        token.getIdentifier());
    tokenId.setRenewer(new Text("om"));
    tokenId.setMaxDate(System.currentTimeMillis() * 2);
    Token<OzoneTokenIdentifier> tamperedToken = new Token<>(
        tokenId.getBytes(), token2.getPassword(), token2.getKind(),
        token2.getService());
    LambdaTestUtils.intercept(OMException.class,
        "Delegation token renewal failed",
        () -> omClient.renewDelegationToken(tamperedToken));
    assertTrue(omLogs.getOutput().contains("can't be found in " +
        "cache"));
    omLogs.clearOutput();

  } finally {
    om.stop();
    om.join();
  }
}
 

/**
 * Tests functionality to init secure OM when it is already initialized.
 */
@Test
public void testSecureOmReInit() throws Exception {
  LogCapturer omLogs =
      LogCapturer.captureLogs(OzoneManager.getLogger());
  omLogs.clearOutput();

  /*
   * As all these processes run inside the same JVM, there are issues around
   * the Hadoop UGI if different processes run with different principals.
   * In this test, the OM has to contact the SCM to download certs. SCM runs
   * as scm/host@REALM, but the OM logs in as om/host@REALM, and then the test
   * fails, and the OM is unable to contact the SCM due to kerberos login
   * issues. To work around that, have the OM run as the same principal as the
   * SCM, and then the test passes.
   *
   * TODO: Need to look into this further to see if there is a better way to
   *       address this problem.
   */
  String realm = miniKdc.getRealm();
  conf.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY,
      "scm/" + host + "@" + realm);
  omKeyTab = new File(workDir, "scm.keytab");
  conf.set(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
      omKeyTab.getAbsolutePath());

  initSCM();
  try {
    scm = HddsTestUtils.getScm(conf);
    scm.start();
    conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, false);
    OMStorage omStore = new OMStorage(conf);
    initializeOmStorage(omStore);
    OzoneManager.setTestSecureOmFlag(true);
    om = OzoneManager.createOm(conf);

    assertNull(om.getCertificateClient());
    assertFalse(omLogs.getOutput().contains("Init response: GETCERT"));
    assertFalse(omLogs.getOutput().contains("Successfully stored " +
        "SCM signed certificate"));

    conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
    OzoneManager.omInit(conf);
    om.stop();
    om = OzoneManager.createOm(conf);

    assertNotNull(om.getCertificateClient());
    assertNotNull(om.getCertificateClient().getPublicKey());
    assertNotNull(om.getCertificateClient().getPrivateKey());
    assertNotNull(om.getCertificateClient().getCertificate());
    assertTrue(omLogs.getOutput().contains("Init response: GETCERT"));
    assertTrue(omLogs.getOutput().contains("Successfully stored " +
        "SCM signed certificate"));
    X509Certificate certificate = om.getCertificateClient().getCertificate();
    validateCertificate(certificate);

  } finally {
    if (scm != null) {
      scm.stop();
    }
  }

}