Java Code Examples for org.apache.hadoop.hdfs.HAUtil#isHAEnabled()

The following examples show how to use org.apache.hadoop.hdfs.HAUtil#isHAEnabled() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: FSImage.java From hadoop with Apache License 2.0

6 votes

/** rollback for rolling upgrade. */
private void rollingRollback(long discardSegmentTxId, long ckptId)
    throws IOException {
  // discard discard unnecessary editlog segments starting from the given id
  this.editLog.discardSegments(discardSegmentTxId);
  // rename the special checkpoint
  renameCheckpoint(ckptId, NameNodeFile.IMAGE_ROLLBACK, NameNodeFile.IMAGE,
      true);
  // purge all the checkpoints after the marker
  archivalManager.purgeCheckpoinsAfter(NameNodeFile.IMAGE, ckptId);
  String nameserviceId = DFSUtil.getNamenodeNameServiceId(conf);
  if (HAUtil.isHAEnabled(conf, nameserviceId)) {
    // close the editlog since it is currently open for write
    this.editLog.close();
    // reopen the editlog for read
    this.editLog.initSharedJournalsForRead();
  }
}

Example 2

Source File: DFSZKFailoverController.java From hadoop with Apache License 2.0

6 votes

public static DFSZKFailoverController create(Configuration conf) {
  Configuration localNNConf = DFSHAAdmin.addSecurityConfiguration(conf);
  String nsId = DFSUtil.getNamenodeNameServiceId(conf);

  if (!HAUtil.isHAEnabled(localNNConf, nsId)) {
    throw new HadoopIllegalArgumentException(
        "HA is not enabled for this namenode.");
  }
  String nnId = HAUtil.getNameNodeId(localNNConf, nsId);
  if (nnId == null) {
    String msg = "Could not get the namenode ID of this node. " +
        "You may run zkfc on the node other than namenode.";
    throw new HadoopIllegalArgumentException(msg);
  }
  NameNode.initializeGenericKeys(localNNConf, nsId, nnId);
  DFSUtil.setGenericConf(localNNConf, nsId, nnId, ZKFC_CONF_KEYS);
  
  NNHAServiceTarget localTarget = new NNHAServiceTarget(
      localNNConf, nsId, nnId);
  return new DFSZKFailoverController(localNNConf, localTarget);
}

Example 3

Source File: YarnUtils.java From twill with Apache License 2.0

6 votes

/**
 * Clones the delegation token to individual host behind the same logical address.
 *
 * @param config the hadoop configuration
 * @throws IOException if failed to get information for the current user.
 */
public static void cloneHaNnCredentials(Configuration config) throws IOException {
  String scheme = URI.create(config.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY,
                                        CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT)).getScheme();

  // Loop through all name services. Each name service could have multiple name node associated with it.
  for (Map.Entry<String, Map<String, InetSocketAddress>> entry : getHaNnRpcAddresses(config).entrySet()) {
    String nsId = entry.getKey();
    Map<String, InetSocketAddress> addressesInNN = entry.getValue();
    if (!HAUtil.isHAEnabled(config, nsId) || addressesInNN == null || addressesInNN.isEmpty()) {
      continue;
    }

    // The client may have a delegation token set for the logical
    // URI of the cluster. Clone this token to apply to each of the
    // underlying IPC addresses so that the IPC code can find it.
    URI uri = URI.create(scheme + "://" + nsId);

    LOG.info("Cloning delegation token for uri {}", uri);
    cloneDelegationTokenForLogicalUri(UserGroupInformation.getCurrentUser(), uri, addressesInNN.values());
  }
}

Example 4

Source File: FSImage.java From big-c with Apache License 2.0

6 votes

/** rollback for rolling upgrade. */
private void rollingRollback(long discardSegmentTxId, long ckptId)
    throws IOException {
  // discard discard unnecessary editlog segments starting from the given id
  this.editLog.discardSegments(discardSegmentTxId);
  // rename the special checkpoint
  renameCheckpoint(ckptId, NameNodeFile.IMAGE_ROLLBACK, NameNodeFile.IMAGE,
      true);
  // purge all the checkpoints after the marker
  archivalManager.purgeCheckpoinsAfter(NameNodeFile.IMAGE, ckptId);
  String nameserviceId = DFSUtil.getNamenodeNameServiceId(conf);
  if (HAUtil.isHAEnabled(conf, nameserviceId)) {
    // close the editlog since it is currently open for write
    this.editLog.close();
    // reopen the editlog for read
    this.editLog.initSharedJournalsForRead();
  }
}

Example 5

Source File: DFSZKFailoverController.java From big-c with Apache License 2.0

6 votes

public static DFSZKFailoverController create(Configuration conf) {
  Configuration localNNConf = DFSHAAdmin.addSecurityConfiguration(conf);
  String nsId = DFSUtil.getNamenodeNameServiceId(conf);

  if (!HAUtil.isHAEnabled(localNNConf, nsId)) {
    throw new HadoopIllegalArgumentException(
        "HA is not enabled for this namenode.");
  }
  String nnId = HAUtil.getNameNodeId(localNNConf, nsId);
  if (nnId == null) {
    String msg = "Could not get the namenode ID of this node. " +
        "You may run zkfc on the node other than namenode.";
    throw new HadoopIllegalArgumentException(msg);
  }
  NameNode.initializeGenericKeys(localNNConf, nsId, nnId);
  DFSUtil.setGenericConf(localNNConf, nsId, nnId, ZKFC_CONF_KEYS);
  
  NNHAServiceTarget localTarget = new NNHAServiceTarget(
      localNNConf, nsId, nnId);
  return new DFSZKFailoverController(localNNConf, localTarget);
}

Example 6

Source File: BootstrapStandby.java From hadoop with Apache License 2.0

5 votes

private void parseConfAndFindOtherNN() throws IOException {
  Configuration conf = getConf();
  nsId = DFSUtil.getNamenodeNameServiceId(conf);

  if (!HAUtil.isHAEnabled(conf, nsId)) {
    throw new HadoopIllegalArgumentException(
        "HA is not enabled for this namenode.");
  }
  nnId = HAUtil.getNameNodeId(conf, nsId);
  NameNode.initializeGenericKeys(conf, nsId, nnId);

  if (!HAUtil.usesSharedEditsDir(conf)) {
    throw new HadoopIllegalArgumentException(
      "Shared edits storage is not enabled for this namenode.");
  }
  
  Configuration otherNode = HAUtil.getConfForOtherNode(conf);
  otherNNId = HAUtil.getNameNodeId(otherNode, nsId);
  otherIpcAddr = NameNode.getServiceAddress(otherNode, true);
  Preconditions.checkArgument(otherIpcAddr.getPort() != 0 &&
      !otherIpcAddr.getAddress().isAnyLocalAddress(),
      "Could not determine valid IPC address for other NameNode (%s)" +
      ", got: %s", otherNNId, otherIpcAddr);

  final String scheme = DFSUtil.getHttpClientScheme(conf);
  otherHttpAddr = DFSUtil.getInfoServerWithDefaultHost(
      otherIpcAddr.getHostName(), otherNode, scheme).toURL();

  dirsToFormat = FSNamesystem.getNamespaceDirs(conf);
  editUrisToFormat = FSNamesystem.getNamespaceEditsDirs(
      conf, false);
  sharedEditsUris = FSNamesystem.getSharedEditsDirs(conf);
}

Example 7

Source File: FSImage.java From hadoop with Apache License 2.0

5 votes

public void initEditLog(StartupOption startOpt) throws IOException {
  Preconditions.checkState(getNamespaceID() != 0,
      "Must know namespace ID before initting edit log");
  String nameserviceId = DFSUtil.getNamenodeNameServiceId(conf);
  if (!HAUtil.isHAEnabled(conf, nameserviceId)) {
    // If this NN is not HA
    editLog.initJournalsForWrite();
    editLog.recoverUnclosedStreams();
  } else if (HAUtil.isHAEnabled(conf, nameserviceId)
      && (startOpt == StartupOption.UPGRADE
          || startOpt == StartupOption.UPGRADEONLY
          || RollingUpgradeStartupOption.ROLLBACK.matches(startOpt))) {
    // This NN is HA, but we're doing an upgrade or a rollback of rolling
    // upgrade so init the edit log for write.
    editLog.initJournalsForWrite();
    if (startOpt == StartupOption.UPGRADE
        || startOpt == StartupOption.UPGRADEONLY) {
      long sharedLogCTime = editLog.getSharedLogCTime();
      if (this.storage.getCTime() < sharedLogCTime) {
        throw new IOException("It looks like the shared log is already " +
            "being upgraded but this NN has not been upgraded yet. You " +
            "should restart this NameNode with the '" +
            StartupOption.BOOTSTRAPSTANDBY.getName() + "' option to bring " +
            "this NN in sync with the other.");
      }
    }
    editLog.recoverUnclosedStreams();
  } else {
    // This NN is HA and we're not doing an upgrade.
    editLog.initSharedJournalsForRead();
  }
}

Example 8

Source File: BootstrapStandby.java From big-c with Apache License 2.0

5 votes

private void parseConfAndFindOtherNN() throws IOException {
  Configuration conf = getConf();
  nsId = DFSUtil.getNamenodeNameServiceId(conf);

  if (!HAUtil.isHAEnabled(conf, nsId)) {
    throw new HadoopIllegalArgumentException(
        "HA is not enabled for this namenode.");
  }
  nnId = HAUtil.getNameNodeId(conf, nsId);
  NameNode.initializeGenericKeys(conf, nsId, nnId);

  if (!HAUtil.usesSharedEditsDir(conf)) {
    throw new HadoopIllegalArgumentException(
      "Shared edits storage is not enabled for this namenode.");
  }
  
  Configuration otherNode = HAUtil.getConfForOtherNode(conf);
  otherNNId = HAUtil.getNameNodeId(otherNode, nsId);
  otherIpcAddr = NameNode.getServiceAddress(otherNode, true);
  Preconditions.checkArgument(otherIpcAddr.getPort() != 0 &&
      !otherIpcAddr.getAddress().isAnyLocalAddress(),
      "Could not determine valid IPC address for other NameNode (%s)" +
      ", got: %s", otherNNId, otherIpcAddr);

  final String scheme = DFSUtil.getHttpClientScheme(conf);
  otherHttpAddr = DFSUtil.getInfoServerWithDefaultHost(
      otherIpcAddr.getHostName(), otherNode, scheme).toURL();

  dirsToFormat = FSNamesystem.getNamespaceDirs(conf);
  editUrisToFormat = FSNamesystem.getNamespaceEditsDirs(
      conf, false);
  sharedEditsUris = FSNamesystem.getSharedEditsDirs(conf);
}

Example 9

Source File: FSImage.java From big-c with Apache License 2.0

5 votes

public void initEditLog(StartupOption startOpt) throws IOException {
  Preconditions.checkState(getNamespaceID() != 0,
      "Must know namespace ID before initting edit log");
  String nameserviceId = DFSUtil.getNamenodeNameServiceId(conf);
  if (!HAUtil.isHAEnabled(conf, nameserviceId)) {
    // If this NN is not HA
    editLog.initJournalsForWrite();
    editLog.recoverUnclosedStreams();
  } else if (HAUtil.isHAEnabled(conf, nameserviceId)
      && (startOpt == StartupOption.UPGRADE
          || startOpt == StartupOption.UPGRADEONLY
          || RollingUpgradeStartupOption.ROLLBACK.matches(startOpt))) {
    // This NN is HA, but we're doing an upgrade or a rollback of rolling
    // upgrade so init the edit log for write.
    editLog.initJournalsForWrite();
    if (startOpt == StartupOption.UPGRADE
        || startOpt == StartupOption.UPGRADEONLY) {
      long sharedLogCTime = editLog.getSharedLogCTime();
      if (this.storage.getCTime() < sharedLogCTime) {
        throw new IOException("It looks like the shared log is already " +
            "being upgraded but this NN has not been upgraded yet. You " +
            "should restart this NameNode with the '" +
            StartupOption.BOOTSTRAPSTANDBY.getName() + "' option to bring " +
            "this NN in sync with the other.");
      }
    }
    editLog.recoverUnclosedStreams();
  } else {
    // This NN is HA and we're not doing an upgrade.
    editLog.initSharedJournalsForRead();
  }
}

Example 10

Source File: ImageServlet.java From hadoop with Apache License 2.0

4 votes

@VisibleForTesting
static boolean isValidRequestor(ServletContext context, String remoteUser,
    Configuration conf) throws IOException {
  if (remoteUser == null) { // This really shouldn't happen...
    LOG.warn("Received null remoteUser while authorizing access to getImage servlet");
    return false;
  }

  Set<String> validRequestors = new HashSet<String>();

  validRequestors.add(SecurityUtil.getServerPrincipal(conf
      .get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY),
      NameNode.getAddress(conf).getHostName()));
  try {
    validRequestors.add(
        SecurityUtil.getServerPrincipal(conf
            .get(DFSConfigKeys.DFS_SECONDARY_NAMENODE_KERBEROS_PRINCIPAL_KEY),
            SecondaryNameNode.getHttpAddress(conf).getHostName()));
  } catch (Exception e) {
    // Don't halt if SecondaryNameNode principal could not be added.
    LOG.debug("SecondaryNameNode principal could not be added", e);
    String msg = String.format(
      "SecondaryNameNode principal not considered, %s = %s, %s = %s",
      DFSConfigKeys.DFS_SECONDARY_NAMENODE_KERBEROS_PRINCIPAL_KEY,
      conf.get(DFSConfigKeys.DFS_SECONDARY_NAMENODE_KERBEROS_PRINCIPAL_KEY),
      DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,
      conf.getTrimmed(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,
        DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_DEFAULT));
    LOG.warn(msg);
  }

  if (HAUtil.isHAEnabled(conf, DFSUtil.getNamenodeNameServiceId(conf))) {
    Configuration otherNnConf = HAUtil.getConfForOtherNode(conf);
    validRequestors.add(SecurityUtil.getServerPrincipal(otherNnConf
        .get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY),
        NameNode.getAddress(otherNnConf).getHostName()));
  }

  for (String v : validRequestors) {
    if (v != null && v.equals(remoteUser)) {
      LOG.info("ImageServlet allowing checkpointer: " + remoteUser);
      return true;
    }
  }

  if (HttpServer2.userHasAdministratorAccess(context, remoteUser)) {
    LOG.info("ImageServlet allowing administrator: " + remoteUser);
    return true;
  }

  LOG.info("ImageServlet rejecting: " + remoteUser);
  return false;
}

Example 11

Source File: BlockManager.java From hadoop with Apache License 2.0

4 votes

private static BlockTokenSecretManager createBlockTokenSecretManager(
    final Configuration conf) {
  final boolean isEnabled = conf.getBoolean(
      DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, 
      DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_DEFAULT);
  LOG.info(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY + "=" + isEnabled);

  if (!isEnabled) {
    if (UserGroupInformation.isSecurityEnabled()) {
      LOG.error("Security is enabled but block access tokens " +
          "(via " + DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY + ") " +
          "aren't enabled. This may cause issues " +
          "when clients attempt to talk to a DataNode.");
    }
    return null;
  }

  final long updateMin = conf.getLong(
      DFSConfigKeys.DFS_BLOCK_ACCESS_KEY_UPDATE_INTERVAL_KEY, 
      DFSConfigKeys.DFS_BLOCK_ACCESS_KEY_UPDATE_INTERVAL_DEFAULT);
  final long lifetimeMin = conf.getLong(
      DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_LIFETIME_KEY, 
      DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_LIFETIME_DEFAULT);
  final String encryptionAlgorithm = conf.get(
      DFSConfigKeys.DFS_DATA_ENCRYPTION_ALGORITHM_KEY);
  LOG.info(DFSConfigKeys.DFS_BLOCK_ACCESS_KEY_UPDATE_INTERVAL_KEY
      + "=" + updateMin + " min(s), "
      + DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_LIFETIME_KEY
      + "=" + lifetimeMin + " min(s), "
      + DFSConfigKeys.DFS_DATA_ENCRYPTION_ALGORITHM_KEY
      + "=" + encryptionAlgorithm);
  
  String nsId = DFSUtil.getNamenodeNameServiceId(conf);
  boolean isHaEnabled = HAUtil.isHAEnabled(conf, nsId);

  if (isHaEnabled) {
    String thisNnId = HAUtil.getNameNodeId(conf, nsId);
    String otherNnId = HAUtil.getNameNodeIdOfOtherNode(conf, nsId);
    return new BlockTokenSecretManager(updateMin*60*1000L,
        lifetimeMin*60*1000L, thisNnId.compareTo(otherNnId) < 0 ? 0 : 1, null,
        encryptionAlgorithm);
  } else {
    return new BlockTokenSecretManager(updateMin*60*1000L,
        lifetimeMin*60*1000L, 0, null, encryptionAlgorithm);
  }
}

Example 12

Source File: ImageServlet.java From big-c with Apache License 2.0

4 votes

@VisibleForTesting
static boolean isValidRequestor(ServletContext context, String remoteUser,
    Configuration conf) throws IOException {
  if (remoteUser == null) { // This really shouldn't happen...
    LOG.warn("Received null remoteUser while authorizing access to getImage servlet");
    return false;
  }

  Set<String> validRequestors = new HashSet<String>();

  validRequestors.add(SecurityUtil.getServerPrincipal(conf
      .get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY),
      NameNode.getAddress(conf).getHostName()));
  try {
    validRequestors.add(
        SecurityUtil.getServerPrincipal(conf
            .get(DFSConfigKeys.DFS_SECONDARY_NAMENODE_KERBEROS_PRINCIPAL_KEY),
            SecondaryNameNode.getHttpAddress(conf).getHostName()));
  } catch (Exception e) {
    // Don't halt if SecondaryNameNode principal could not be added.
    LOG.debug("SecondaryNameNode principal could not be added", e);
    String msg = String.format(
      "SecondaryNameNode principal not considered, %s = %s, %s = %s",
      DFSConfigKeys.DFS_SECONDARY_NAMENODE_KERBEROS_PRINCIPAL_KEY,
      conf.get(DFSConfigKeys.DFS_SECONDARY_NAMENODE_KERBEROS_PRINCIPAL_KEY),
      DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,
      conf.getTrimmed(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,
        DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_DEFAULT));
    LOG.warn(msg);
  }

  if (HAUtil.isHAEnabled(conf, DFSUtil.getNamenodeNameServiceId(conf))) {
    Configuration otherNnConf = HAUtil.getConfForOtherNode(conf);
    validRequestors.add(SecurityUtil.getServerPrincipal(otherNnConf
        .get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY),
        NameNode.getAddress(otherNnConf).getHostName()));
  }

  for (String v : validRequestors) {
    if (v != null && v.equals(remoteUser)) {
      LOG.info("ImageServlet allowing checkpointer: " + remoteUser);
      return true;
    }
  }

  if (HttpServer2.userHasAdministratorAccess(context, remoteUser)) {
    LOG.info("ImageServlet allowing administrator: " + remoteUser);
    return true;
  }

  LOG.info("ImageServlet rejecting: " + remoteUser);
  return false;
}

Example 13

Source File: BlockManager.java From big-c with Apache License 2.0

4 votes

private static BlockTokenSecretManager createBlockTokenSecretManager(
    final Configuration conf) {
  final boolean isEnabled = conf.getBoolean(
      DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, 
      DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_DEFAULT);
  LOG.info(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY + "=" + isEnabled);

  if (!isEnabled) {
    if (UserGroupInformation.isSecurityEnabled()) {
      LOG.error("Security is enabled but block access tokens " +
          "(via " + DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY + ") " +
          "aren't enabled. This may cause issues " +
          "when clients attempt to talk to a DataNode.");
    }
    return null;
  }

  final long updateMin = conf.getLong(
      DFSConfigKeys.DFS_BLOCK_ACCESS_KEY_UPDATE_INTERVAL_KEY, 
      DFSConfigKeys.DFS_BLOCK_ACCESS_KEY_UPDATE_INTERVAL_DEFAULT);
  final long lifetimeMin = conf.getLong(
      DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_LIFETIME_KEY, 
      DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_LIFETIME_DEFAULT);
  final String encryptionAlgorithm = conf.get(
      DFSConfigKeys.DFS_DATA_ENCRYPTION_ALGORITHM_KEY);
  LOG.info(DFSConfigKeys.DFS_BLOCK_ACCESS_KEY_UPDATE_INTERVAL_KEY
      + "=" + updateMin + " min(s), "
      + DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_LIFETIME_KEY
      + "=" + lifetimeMin + " min(s), "
      + DFSConfigKeys.DFS_DATA_ENCRYPTION_ALGORITHM_KEY
      + "=" + encryptionAlgorithm);
  
  String nsId = DFSUtil.getNamenodeNameServiceId(conf);
  boolean isHaEnabled = HAUtil.isHAEnabled(conf, nsId);

  if (isHaEnabled) {
    String thisNnId = HAUtil.getNameNodeId(conf, nsId);
    String otherNnId = HAUtil.getNameNodeIdOfOtherNode(conf, nsId);
    return new BlockTokenSecretManager(updateMin*60*1000L,
        lifetimeMin*60*1000L, thisNnId.compareTo(otherNnId) < 0 ? 0 : 1, null,
        encryptionAlgorithm);
  } else {
    return new BlockTokenSecretManager(updateMin*60*1000L,
        lifetimeMin*60*1000L, 0, null, encryptionAlgorithm);
  }
}