Java Code Examples for org.apache.hadoop.conf.Configuration#getTrimmed()

The following examples show how to use org.apache.hadoop.conf.Configuration#getTrimmed() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: hadoop   File: MicroZookeeperService.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * set up security. this must be done prior to creating
 * the ZK instance, as it sets up JAAS if that has not been done already.
 *
 * @return true if the cluster has security enabled.
 */
public boolean setupSecurity() throws IOException {
  Configuration conf = getConfig();
  String jaasContext = conf.getTrimmed(KEY_REGISTRY_ZKSERVICE_JAAS_CONTEXT);
  secureServer = StringUtils.isNotEmpty(jaasContext);
  if (secureServer) {
    RegistrySecurity.validateContext(jaasContext);
    RegistrySecurity.bindZKToServerJAASContext(jaasContext);
    // policy on failed auth
    System.setProperty(PROP_ZK_ALLOW_FAILED_SASL_CLIENTS,
      conf.get(KEY_ZKSERVICE_ALLOW_FAILED_SASL_CLIENTS,
          "true"));

    //needed so that you can use sasl: strings in the registry
    System.setProperty(RegistryInternalConstants.ZOOKEEPER_AUTH_PROVIDER +".1",
        RegistryInternalConstants.SASLAUTHENTICATION_PROVIDER);
    String serverContext =
        System.getProperty(PROP_ZK_SERVER_SASL_CONTEXT);
    addDiagnostics("Server JAAS context s = %s", serverContext);
    return true;
  } else {
    return false;
  }
}
 
Example 2
Source Project: big-c   File: HAUtil.java    License: Apache License 2.0 6 votes vote down vote up
private static void checkAndSetRMRPCAddress(String prefix, String RMId,
    Configuration conf) {
  String rpcAddressConfKey = null;
  try {
    rpcAddressConfKey = addSuffix(prefix, RMId);
    if (conf.getTrimmed(rpcAddressConfKey) == null) {
      String hostNameConfKey = addSuffix(YarnConfiguration.RM_HOSTNAME, RMId);
      String confVal = conf.getTrimmed(hostNameConfKey);
      if (confVal == null) {
        throwBadConfigurationException(getNeedToSetValueMessage(
            hostNameConfKey + " or " + addSuffix(prefix, RMId)));
      } else {
        conf.set(addSuffix(prefix, RMId), confVal + ":"
            + YarnConfiguration.getRMDefaultPortNumber(prefix, conf));
      }
    }
  } catch (IllegalArgumentException iae) {
    String errmsg = iae.getMessage();
    if (rpcAddressConfKey == null) {
      // Error at addSuffix
      errmsg = getInvalidValueMessage(YarnConfiguration.RM_HA_ID, RMId);
    }
    throwBadConfigurationException(errmsg);
  }
}
 
Example 3
Source Project: tez   File: TaskSpecificLaunchCmdOption.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * <pre>
 * Get the set of tasks in the job, which needs task specific launch command arguments to be
 * added. Example formats are
 * v[0,1,2] - Add additional options to subset of tasks in a vertex
 * v[1,2,3];v2[5,6,7] - Add additional options in tasks in multiple vertices
 * v[1:5,20,30];v2[2:5,60,7] - To support range of tasks in vertices. Partial
 * ranges are not supported (e.g v[:5],v2[2:]).
 * v[] - For all tasks in a vertex
 * </pre>
 *
 * @param conf
 * @return a map from the vertex name to a BitSet representing tasks to be instruemented. null if
 *         the provided configuration is empty or invalid
 */
private Map<String, BitSet> getSpecificTasks(Configuration conf) {
  String specificTaskList =
      conf.getTrimmed(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS_LIST);
  if (!Strings.isNullOrEmpty(specificTaskList) && isValid(specificTaskList)) {
    final Map<String, BitSet> resultSet = new HashMap<String, BitSet>();
    Matcher matcher = TASKS_REGEX.matcher(specificTaskList);
    while (matcher.find()) {
      String vertexName = matcher.group(1).trim();
      BitSet taskSet = parseTasks(matcher.group(2).trim());
      resultSet.put(vertexName, taskSet);
    }
    LOG.info("Specific tasks with additional launch-cmd options=" + resultSet);
    return resultSet;
  } else {
    return null;
  }
}
 
Example 4
Source Project: hadoop   File: HAUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get the namenode Id by matching the {@code addressKey}
 * with the the address of the local node.
 * 
 * If {@link DFSConfigKeys#DFS_HA_NAMENODE_ID_KEY} is not specifically
 * configured, this method determines the namenode Id by matching the local
 * node's address with the configured addresses. When a match is found, it
 * returns the namenode Id from the corresponding configuration key.
 * 
 * @param conf Configuration
 * @return namenode Id on success, null on failure.
 * @throws HadoopIllegalArgumentException on error
 */
public static String getNameNodeId(Configuration conf, String nsId) {
  String namenodeId = conf.getTrimmed(DFS_HA_NAMENODE_ID_KEY);
  if (namenodeId != null) {
    return namenodeId;
  }
  
  String suffixes[] = DFSUtil.getSuffixIDs(conf, DFS_NAMENODE_RPC_ADDRESS_KEY,
      nsId, null, DFSUtil.LOCAL_ADDRESS_MATCHER);
  if (suffixes == null) {
    String msg = "Configuration " + DFS_NAMENODE_RPC_ADDRESS_KEY + 
        " must be suffixed with nameservice and namenode ID for HA " +
        "configuration.";
    throw new HadoopIllegalArgumentException(msg);
  }
  
  return suffixes[1];
}
 
Example 5
Source Project: big-c   File: MicroZookeeperService.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * set up security. this must be done prior to creating
 * the ZK instance, as it sets up JAAS if that has not been done already.
 *
 * @return true if the cluster has security enabled.
 */
public boolean setupSecurity() throws IOException {
  Configuration conf = getConfig();
  String jaasContext = conf.getTrimmed(KEY_REGISTRY_ZKSERVICE_JAAS_CONTEXT);
  secureServer = StringUtils.isNotEmpty(jaasContext);
  if (secureServer) {
    RegistrySecurity.validateContext(jaasContext);
    RegistrySecurity.bindZKToServerJAASContext(jaasContext);
    // policy on failed auth
    System.setProperty(PROP_ZK_ALLOW_FAILED_SASL_CLIENTS,
      conf.get(KEY_ZKSERVICE_ALLOW_FAILED_SASL_CLIENTS,
          "true"));

    //needed so that you can use sasl: strings in the registry
    System.setProperty(RegistryInternalConstants.ZOOKEEPER_AUTH_PROVIDER +".1",
        RegistryInternalConstants.SASLAUTHENTICATION_PROVIDER);
    String serverContext =
        System.getProperty(PROP_ZK_SERVER_SASL_CONTEXT);
    addDiagnostics("Server JAAS context s = %s", serverContext);
    return true;
  } else {
    return false;
  }
}
 
Example 6
Source Project: big-c   File: CuratorService.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Init the service.
 * This is where the security bindings are set up
 * @param conf configuration of the service
 * @throws Exception
 */
@Override
protected void serviceInit(Configuration conf) throws Exception {

  registryRoot = conf.getTrimmed(KEY_REGISTRY_ZK_ROOT,
      DEFAULT_ZK_REGISTRY_ROOT);

  // create and add the registy service
  registrySecurity = new RegistrySecurity("registry security");
  addService(registrySecurity);

  if (LOG.isDebugEnabled()) {
    LOG.debug("Creating Registry with root {}", registryRoot);
  }

  super.serviceInit(conf);
}
 
Example 7
public static ProxyConfiguration initProxySupport(Configuration conf) throws IllegalArgumentException {
  final ProxyConfiguration.Builder builder = ProxyConfiguration.builder();

  final String proxyHost = conf.getTrimmed(Constants.PROXY_HOST, "");
  int proxyPort = conf.getInt(Constants.PROXY_PORT, -1);
  if (!proxyHost.isEmpty()) {
    if (proxyPort < 0) {
      if (conf.getBoolean(Constants.SECURE_CONNECTIONS, Constants.DEFAULT_SECURE_CONNECTIONS)) {
        proxyPort = 443;
      } else {
        proxyPort = 80;
      }
    }

    builder.endpoint(URI.create(proxyHost + ":" + proxyPort));

    try {
      final String proxyUsername = lookupPassword(conf, Constants.PROXY_USERNAME);
      final String proxyPassword = lookupPassword(conf, Constants.PROXY_PASSWORD);
      if ((proxyUsername == null) != (proxyPassword == null)) {
        throw new IllegalArgumentException(String.format("Proxy error: %s or %s set without the other.",
          Constants.PROXY_USERNAME, Constants.PROXY_PASSWORD));
      }

      builder.username(proxyUsername);
      builder.password(proxyPassword);
      builder.ntlmDomain(conf.getTrimmed(Constants.PROXY_DOMAIN));
      builder.ntlmWorkstation(conf.getTrimmed(Constants.PROXY_WORKSTATION));
    } catch (IOException e) {
      throw UserException.sourceInBadState(e).buildSilently();
    }
  } else if (proxyPort >= 0) {
    throw new IllegalArgumentException(String.format("Proxy error: %s set without %s",
      Constants.PROXY_HOST, Constants.PROXY_PORT));
  }

  return builder.build();
}
 
Example 8
Source Project: tez   File: TaskSpecificLaunchCmdOption.java    License: Apache License 2.0 5 votes vote down vote up
public TaskSpecificLaunchCmdOption(Configuration conf) {
  this.tsLaunchCmdOpts =
      conf.getTrimmed(TezConfiguration.TEZ_TASK_SPECIFIC_LAUNCH_CMD_OPTS);
  this.tsLogParams = TezClientUtils
      .parseLogParams(conf.getTrimmed(TezConfiguration.TEZ_TASK_SPECIFIC_LOG_LEVEL));

  if (shouldParseSpecificTaskList()) {
    this.tasksMap = getSpecificTasks(conf);
  } else {
    this.tasksMap = null;
  }
}
 
Example 9
Source Project: big-c   File: DataNode.java    License: Apache License 2.0 5 votes vote down vote up
static DomainPeerServer getDomainPeerServer(Configuration conf,
    int port) throws IOException {
  String domainSocketPath =
      conf.getTrimmed(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY,
          DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_DEFAULT);
  if (domainSocketPath.isEmpty()) {
    if (conf.getBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY,
          DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_DEFAULT) &&
       (!conf.getBoolean(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL,
        DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL_DEFAULT))) {
      LOG.warn("Although short-circuit local reads are configured, " +
          "they are disabled because you didn't configure " +
          DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY);
    }
    return null;
  }
  if (DomainSocket.getLoadingFailureReason() != null) {
    throw new RuntimeException("Although a UNIX domain socket " +
        "path is configured as " + domainSocketPath + ", we cannot " +
        "start a localDataXceiverServer because " +
        DomainSocket.getLoadingFailureReason());
  }
  DomainPeerServer domainPeerServer =
    new DomainPeerServer(domainSocketPath, port);
  domainPeerServer.setReceiveBufferSize(
      HdfsConstants.DEFAULT_DATA_SOCKET_SIZE);
  return domainPeerServer;
}
 
Example 10
Source Project: hadoop   File: HAUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @param conf Configuration. Please use verifyAndSetRMHAId to check.
 * @return RM Id on success
 */
public static String getRMHAId(Configuration conf) {
  int found = 0;
  String currentRMId = conf.getTrimmed(YarnConfiguration.RM_HA_ID);
  if(currentRMId == null) {
    for(String rmId : getRMHAIds(conf)) {
      String key = addSuffix(YarnConfiguration.RM_ADDRESS, rmId);
      String addr = conf.get(key);
      if (addr == null) {
        continue;
      }
      InetSocketAddress s;
      try {
        s = NetUtils.createSocketAddr(addr);
      } catch (Exception e) {
        LOG.warn("Exception in creating socket address " + addr, e);
        continue;
      }
      if (!s.isUnresolved() && NetUtils.isLocalAddress(s.getAddress())) {
        currentRMId = rmId.trim();
        found++;
      }
    }
  }
  if (found > 1) { // Only one address must match the local address
    String msg = "The HA Configuration has multiple addresses that match "
        + "local node's address.";
    throw new HadoopIllegalArgumentException(msg);
  }
  return currentRMId;
}
 
Example 11
Source Project: hadoop   File: HAUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static String getConfValueForRMInstance(String prefix,
                                               Configuration conf) {
  String confKey = getConfKeyForRMInstance(prefix, conf);
  String retVal = conf.getTrimmed(confKey);
  if (LOG.isTraceEnabled()) {
    LOG.trace("getConfValueForRMInstance: prefix = " + prefix +
        "; confKey being looked up = " + confKey +
        "; value being set to = " + retVal);
  }
  return retVal;
}
 
Example 12
Source Project: big-c   File: HAUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static String getConfValueForRMInstance(String prefix,
                                               Configuration conf) {
  String confKey = getConfKeyForRMInstance(prefix, conf);
  String retVal = conf.getTrimmed(confKey);
  if (LOG.isTraceEnabled()) {
    LOG.trace("getConfValueForRMInstance: prefix = " + prefix +
        "; confKey being looked up = " + confKey +
        "; value being set to = " + retVal);
  }
  return retVal;
}
 
Example 13
public JavaProfilerOptions(Configuration conf) {
  jvmProfilingOpts =
      conf.getTrimmed(TezConfiguration.TEZ_PROFILE_JVM_OPTS, "");

  if (!Strings.isNullOrEmpty(jvmProfilingOpts)) {
    this.tasksToProfileMap = getTasksToProfile(conf);
    if (!tasksToProfileMap.isEmpty() && jvmProfilingOpts.isEmpty()) {
      LOG.warn(TezConfiguration.TEZ_PROFILE_JVM_OPTS
          + " should be specified for profiling");
    }
  }
}
 
Example 14
Source Project: big-c   File: NameNode.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Fetches the address for services to use when connecting to namenode
 * based on the value of fallback returns null if the special
 * address is not specified or returns the default namenode address
 * to be used by both clients and services.
 * Services here are datanodes, backup node, any non client connection
 */
public static InetSocketAddress getServiceAddress(Configuration conf,
                                                      boolean fallback) {
  String addr = conf.getTrimmed(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY);
  if (addr == null || addr.isEmpty()) {
    return fallback ? getAddress(conf) : null;
  }
  return getAddress(addr);
}
 
Example 15
Source Project: hadoop   File: NameNode.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Gets a trimmed value from configuration, or null if no value is defined.
 *
 * @param conf configuration
 * @param key configuration key to get
 * @return trimmed value, or null if no value is defined
 */
private static String getTrimmedOrNull(Configuration conf, String key) {
  String addr = conf.getTrimmed(key);
  if (addr == null || addr.isEmpty()) {
    return null;
  }
  return addr;
}
 
Example 16
Source Project: big-c   File: NameNode.java    License: Apache License 2.0 5 votes vote down vote up
/** Given a configuration get the bind host of the client rpc server
 *  If the bind host is not configured returns null.
 */
protected String getRpcServerBindHost(Configuration conf) {
  String addr = conf.getTrimmed(DFS_NAMENODE_RPC_BIND_HOST_KEY);
  if (addr == null || addr.isEmpty()) {
    return null;
  }
  return addr;
}
 
Example 17
Source Project: hadoop   File: ContainerManagerImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected void serviceStart() throws Exception {

  // Enqueue user dirs in deletion context

  Configuration conf = getConfig();
  final InetSocketAddress initialAddress = conf.getSocketAddr(
      YarnConfiguration.NM_BIND_HOST,
      YarnConfiguration.NM_ADDRESS,
      YarnConfiguration.DEFAULT_NM_ADDRESS,
      YarnConfiguration.DEFAULT_NM_PORT);
  boolean usingEphemeralPort = (initialAddress.getPort() == 0);
  if (context.getNMStateStore().canRecover() && usingEphemeralPort) {
    throw new IllegalArgumentException("Cannot support recovery with an "
        + "ephemeral server port. Check the setting of "
        + YarnConfiguration.NM_ADDRESS);
  }
  // If recovering then delay opening the RPC service until the recovery
  // of resources and containers have completed, otherwise requests from
  // clients during recovery can interfere with the recovery process.
  final boolean delayedRpcServerStart =
      context.getNMStateStore().canRecover();

  Configuration serverConf = new Configuration(conf);

  // always enforce it to be token-based.
  serverConf.set(
    CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    SaslRpcServer.AuthMethod.TOKEN.toString());
  
  YarnRPC rpc = YarnRPC.create(conf);

  server =
      rpc.getServer(ContainerManagementProtocol.class, this, initialAddress, 
          serverConf, this.context.getNMTokenSecretManager(),
          conf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT, 
              YarnConfiguration.DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT));
  
  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
      false)) {
    refreshServiceAcls(conf, new NMPolicyProvider());
  }
  
  LOG.info("Blocking new container-requests as container manager rpc" +
  		" server is still starting.");
  this.setBlockNewContainerRequests(true);

  String bindHost = conf.get(YarnConfiguration.NM_BIND_HOST);
  String nmAddress = conf.getTrimmed(YarnConfiguration.NM_ADDRESS);
  String hostOverride = null;
  if (bindHost != null && !bindHost.isEmpty()
      && nmAddress != null && !nmAddress.isEmpty()) {
    //a bind-host case with an address, to support overriding the first
    //hostname found when querying for our hostname with the specified
    //address, combine the specified address with the actual port listened
    //on by the server
    hostOverride = nmAddress.split(":")[0];
  }

  // setup node ID
  InetSocketAddress connectAddress;
  if (delayedRpcServerStart) {
    connectAddress = NetUtils.getConnectAddress(initialAddress);
  } else {
    server.start();
    connectAddress = NetUtils.getConnectAddress(server);
  }
  NodeId nodeId = buildNodeId(connectAddress, hostOverride);
  ((NodeManager.NMContext)context).setNodeId(nodeId);
  this.context.getNMTokenSecretManager().setNodeId(nodeId);
  this.context.getContainerTokenSecretManager().setNodeId(nodeId);

  // start remaining services
  super.serviceStart();

  if (delayedRpcServerStart) {
    waitForRecoveredContainers();
    server.start();

    // check that the node ID is as previously advertised
    connectAddress = NetUtils.getConnectAddress(server);
    NodeId serverNode = buildNodeId(connectAddress, hostOverride);
    if (!serverNode.equals(nodeId)) {
      throw new IOException("Node mismatch after server started, expected '"
          + nodeId + "' but found '" + serverNode + "'");
    }
  }

  LOG.info("ContainerManager started at " + connectAddress);
  LOG.info("ContainerManager bound to " + initialAddress);
}
 
Example 18
Source Project: big-c   File: SecondaryNameNode.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Initialize SecondaryNameNode.
 */
private void initialize(final Configuration conf,
    CommandLineOpts commandLineOpts) throws IOException {
  final InetSocketAddress infoSocAddr = getHttpAddress(conf);
  final String infoBindAddress = infoSocAddr.getHostName();
  UserGroupInformation.setConfiguration(conf);
  if (UserGroupInformation.isSecurityEnabled()) {
    SecurityUtil.login(conf,
        DFSConfigKeys.DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY,
        DFSConfigKeys.DFS_SECONDARY_NAMENODE_KERBEROS_PRINCIPAL_KEY, infoBindAddress);
  }
  // initiate Java VM metrics
  DefaultMetricsSystem.initialize("SecondaryNameNode");
  JvmMetrics.create("SecondaryNameNode",
      conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY),
      DefaultMetricsSystem.instance());

  // Create connection to the namenode.
  shouldRun = true;
  nameNodeAddr = NameNode.getServiceAddress(conf, true);

  this.conf = conf;
  this.namenode = NameNodeProxies.createNonHAProxy(conf, nameNodeAddr, 
      NamenodeProtocol.class, UserGroupInformation.getCurrentUser(),
      true).getProxy();

  // initialize checkpoint directories
  fsName = getInfoServer();
  checkpointDirs = FSImage.getCheckpointDirs(conf,
                                "/tmp/hadoop/dfs/namesecondary");
  checkpointEditsDirs = FSImage.getCheckpointEditsDirs(conf, 
                                "/tmp/hadoop/dfs/namesecondary");    
  checkpointImage = new CheckpointStorage(conf, checkpointDirs, checkpointEditsDirs);
  checkpointImage.recoverCreate(commandLineOpts.shouldFormat());
  checkpointImage.deleteTempEdits();
  
  namesystem = new FSNamesystem(conf, checkpointImage, true);

  // Disable quota checks
  namesystem.dir.disableQuotaChecks();

  // Initialize other scheduling parameters from the configuration
  checkpointConf = new CheckpointConf(conf);

  final InetSocketAddress httpAddr = infoSocAddr;

  final String httpsAddrString = conf.getTrimmed(
      DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_KEY,
      DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_DEFAULT);
  InetSocketAddress httpsAddr = NetUtils.createSocketAddr(httpsAddrString);

  HttpServer2.Builder builder = DFSUtil.httpServerTemplateForNNAndJN(conf,
      httpAddr, httpsAddr, "secondary",
      DFSConfigKeys.DFS_SECONDARY_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY,
      DFSConfigKeys.DFS_SECONDARY_NAMENODE_KEYTAB_FILE_KEY);

  nameNodeStatusBeanName = MBeans.register("SecondaryNameNode",
          "SecondaryNameNodeInfo", this);

  infoServer = builder.build();

  infoServer.setAttribute("secondary.name.node", this);
  infoServer.setAttribute("name.system.image", checkpointImage);
  infoServer.setAttribute(JspHelper.CURRENT_CONF, conf);
  infoServer.addInternalServlet("imagetransfer", ImageServlet.PATH_SPEC,
      ImageServlet.class, true);
  infoServer.start();

  LOG.info("Web server init done");

  HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf);
  int connIdx = 0;
  if (policy.isHttpEnabled()) {
    InetSocketAddress httpAddress = infoServer.getConnectorAddress(connIdx++);
    conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,
        NetUtils.getHostPortString(httpAddress));
  }

  if (policy.isHttpsEnabled()) {
    InetSocketAddress httpsAddress = infoServer.getConnectorAddress(connIdx);
    conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTPS_ADDRESS_KEY,
        NetUtils.getHostPortString(httpsAddress));
  }

  legacyOivImageDir = conf.get(
      DFSConfigKeys.DFS_NAMENODE_LEGACY_OIV_IMAGE_DIR_KEY);

  LOG.info("Checkpoint Period   :" + checkpointConf.getPeriod() + " secs "
      + "(" + checkpointConf.getPeriod() / 60 + " min)");
  LOG.info("Log Size Trigger    :" + checkpointConf.getTxnCount() + " txns");
}
 
Example 19
Source Project: big-c   File: S3Credentials.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * @throws IllegalArgumentException if credentials for S3 cannot be
 * determined.
 */
public void initialize(URI uri, Configuration conf) {
  if (uri.getHost() == null) {
    throw new IllegalArgumentException("Invalid hostname in URI " + uri);
  }
  
  String userInfo = uri.getUserInfo();
  if (userInfo != null) {
    int index = userInfo.indexOf(':');
    if (index != -1) {
      accessKey = userInfo.substring(0, index);
      secretAccessKey = userInfo.substring(index + 1);
    } else {
      accessKey = userInfo;
    }
  }
  
  String scheme = uri.getScheme();
  String accessKeyProperty = String.format("fs.%s.awsAccessKeyId", scheme);
  String secretAccessKeyProperty =
    String.format("fs.%s.awsSecretAccessKey", scheme);
  if (accessKey == null) {
    accessKey = conf.getTrimmed(accessKeyProperty);
  }
  if (secretAccessKey == null) {
    secretAccessKey = conf.getTrimmed(secretAccessKeyProperty);
  }
  if (accessKey == null && secretAccessKey == null) {
    throw new IllegalArgumentException("AWS " +
                                       "Access Key ID and Secret Access " +
                                       "Key must be specified as the " +
                                       "username or password " +
                                       "(respectively) of a " + scheme +
                                       " URL, or by setting the " +
                                       accessKeyProperty + " or " +
                                       secretAccessKeyProperty +
                                       " properties (respectively).");
  } else if (accessKey == null) {
    throw new IllegalArgumentException("AWS " +
                                       "Access Key ID must be specified " +
                                       "as the username of a " + scheme +
                                       " URL, or by setting the " +
                                       accessKeyProperty + " property.");
  } else if (secretAccessKey == null) {
    throw new IllegalArgumentException("AWS " +
                                       "Secret Access Key must be " +
                                       "specified as the password of a " +
                                       scheme + " URL, or by setting the " +
                                       secretAccessKeyProperty +
                                       " property.");       
  }

}
 
Example 20
Source Project: big-c   File: BackupNode.java    License: Apache License 2.0 4 votes vote down vote up
@Override // NameNode
protected InetSocketAddress getRpcServerAddress(Configuration conf) {
  String addr = conf.getTrimmed(BN_ADDRESS_NAME_KEY, BN_ADDRESS_DEFAULT);
  return NetUtils.createSocketAddr(addr);
}