Java Code Examples for org.apache.hadoop.hbase.HBaseConfiguration#createClusterConf()

The following examples show how to use org.apache.hadoop.hbase.HBaseConfiguration#createClusterConf() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ReplicationUtils.java    From hbase with Apache License 2.0 6 votes vote down vote up
public static Configuration getPeerClusterConfiguration(ReplicationPeerConfig peerConfig,
    Configuration baseConf) throws ReplicationException {
  Configuration otherConf;
  try {
    otherConf = HBaseConfiguration.createClusterConf(baseConf, peerConfig.getClusterKey());
  } catch (IOException e) {
    throw new ReplicationException("Can't get peer configuration for peer " + peerConfig, e);
  }

  if (!peerConfig.getConfiguration().isEmpty()) {
    CompoundConfiguration compound = new CompoundConfiguration();
    compound.add(otherConf);
    compound.addStringMap(peerConfig.getConfiguration());
    return compound;
  }

  return otherConf;
}
 
Example 2
Source File: TableOutputFormat.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public void setConf(Configuration otherConf) {
  String tableName = otherConf.get(OUTPUT_TABLE);
  if(tableName == null || tableName.length() <= 0) {
    throw new IllegalArgumentException("Must specify table name");
  }

  String address = otherConf.get(QUORUM_ADDRESS);
  int zkClientPort = otherConf.getInt(QUORUM_PORT, 0);
  String serverClass = otherConf.get(REGION_SERVER_CLASS);
  String serverImpl = otherConf.get(REGION_SERVER_IMPL);

  try {
    this.conf = HBaseConfiguration.createClusterConf(otherConf, address, OUTPUT_CONF_PREFIX);

    if (serverClass != null) {
      this.conf.set(HConstants.REGION_SERVER_IMPL, serverImpl);
    }
    if (zkClientPort != 0) {
      this.conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
    }
  } catch(IOException e) {
    LOG.error(e.toString(), e);
    throw new RuntimeException(e);
  }
}
 
Example 3
Source File: ReplicationPeerConfigUtil.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Returns the configuration needed to talk to the remote slave cluster.
 * @param conf the base configuration
 * @param peer the description of replication peer
 * @return the configuration for the peer cluster, null if it was unable to get the configuration
 * @throws IOException when create peer cluster configuration failed
 */
public static Configuration getPeerClusterConfiguration(Configuration conf,
    ReplicationPeerDescription peer) throws IOException {
  ReplicationPeerConfig peerConfig = peer.getPeerConfig();
  Configuration otherConf;
  try {
    otherConf = HBaseConfiguration.createClusterConf(conf, peerConfig.getClusterKey());
  } catch (IOException e) {
    throw new IOException("Can't get peer configuration for peerId=" + peer.getPeerId(), e);
  }

  if (!peerConfig.getConfiguration().isEmpty()) {
    CompoundConfiguration compound = new CompoundConfiguration();
    compound.add(otherConf);
    compound.addStringMap(peerConfig.getConfiguration());
    return compound;
  }
  return otherConf;
}
 
Example 4
Source File: VerifyReplication.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void restoreSnapshotForPeerCluster(Configuration conf, String peerQuorumAddress)
  throws IOException {
  Configuration peerConf =
    HBaseConfiguration.createClusterConf(conf, peerQuorumAddress, PEER_CONFIG_PREFIX);
  FileSystem.setDefaultUri(peerConf, peerFSAddress);
  CommonFSUtils.setRootDir(peerConf, new Path(peerFSAddress, peerHBaseRootAddress));
  FileSystem fs = FileSystem.get(peerConf);
  RestoreSnapshotHelper.copySnapshotForScanner(peerConf, fs, CommonFSUtils.getRootDir(peerConf),
    new Path(peerFSAddress, peerSnapshotTmpDir), peerSnapshotName);
}
 
Example 5
Source File: SyncTable.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void initCredentialsForHBase(String zookeeper, Job job) throws IOException {
  Configuration peerConf = HBaseConfiguration.createClusterConf(job
          .getConfiguration(), zookeeper);
  if("kerberos".equalsIgnoreCase(peerConf.get("hbase.security.authentication"))){
    TableMapReduceUtil.initCredentialsForCluster(job, peerConf);
  }
}
 
Example 6
Source File: SyncTable.java    From hbase with Apache License 2.0 5 votes vote down vote up
private static Connection openConnection(Configuration conf, String zkClusterConfKey,
                                         String configPrefix)
  throws IOException {
    String zkCluster = conf.get(zkClusterConfKey);
    Configuration clusterConf = HBaseConfiguration.createClusterConf(conf,
        zkCluster, configPrefix);
    return ConnectionFactory.createConnection(clusterConf);
}
 
Example 7
Source File: TableMapReduceUtil.java    From hbase with Apache License 2.0 5 votes vote down vote up
public static void initCredentials(Job job) throws IOException {
  UserProvider userProvider = UserProvider.instantiate(job.getConfiguration());
  if (userProvider.isHadoopSecurityEnabled()) {
    // propagate delegation related props from launcher job to MR job
    if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
      job.getConfiguration().set("mapreduce.job.credentials.binary",
                                 System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
    }
  }

  if (userProvider.isHBaseSecurityEnabled()) {
    try {
      // init credentials for remote cluster
      String quorumAddress = job.getConfiguration().get(TableOutputFormat.QUORUM_ADDRESS);
      User user = userProvider.getCurrent();
      if (quorumAddress != null) {
        Configuration peerConf = HBaseConfiguration.createClusterConf(job.getConfiguration(),
            quorumAddress, TableOutputFormat.OUTPUT_CONF_PREFIX);
        Connection peerConn = ConnectionFactory.createConnection(peerConf);
        try {
          TokenUtil.addTokenForJob(peerConn, user, job);
        } finally {
          peerConn.close();
        }
      }

      Connection conn = ConnectionFactory.createConnection(job.getConfiguration());
      try {
        TokenUtil.addTokenForJob(conn, user, job);
      } finally {
        conn.close();
      }
    } catch (InterruptedException ie) {
      LOG.info("Interrupted obtaining user authentication token");
      Thread.currentThread().interrupt();
    }
  }
}
 
Example 8
Source File: TestZKConfig.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void testKey(String ensemble, int port, String znode, Boolean multiplePortSupport)
    throws IOException {
  Configuration conf = new Configuration();
  String key = ensemble+":"+port+":"+znode;
  String ensemble2 = null;
  ZKConfig.ZKClusterKey zkClusterKey = ZKConfig.transformClusterKey(key);
  if (multiplePortSupport) {
    ensemble2 = ZKConfig.standardizeZKQuorumServerString(ensemble,
        Integer.toString(port));
    assertEquals(ensemble2, zkClusterKey.getQuorumString());
  }
  else {
    assertEquals(ensemble, zkClusterKey.getQuorumString());
  }
  assertEquals(port, zkClusterKey.getClientPort());
  assertEquals(znode, zkClusterKey.getZnodeParent());

  conf = HBaseConfiguration.createClusterConf(conf, key);
  assertEquals(zkClusterKey.getQuorumString(), conf.get(HConstants.ZOOKEEPER_QUORUM));
  assertEquals(zkClusterKey.getClientPort(), conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, -1));
  assertEquals(zkClusterKey.getZnodeParent(), conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT));

  String reconstructedKey = ZKConfig.getZooKeeperClusterKey(conf);
  if (multiplePortSupport) {
    String key2 = ensemble2 + ":" + port + ":" + znode;
    assertEquals(key2, reconstructedKey);
  }
  else {
    assertEquals(key, reconstructedKey);
  }
}
 
Example 9
Source File: SyncTable.java    From hbase with Apache License 2.0 4 votes vote down vote up
public Job createSubmittableJob(String[] args) throws IOException {
  FileSystem fs = sourceHashDir.getFileSystem(getConf());
  if (!fs.exists(sourceHashDir)) {
    throw new IOException("Source hash dir not found: " + sourceHashDir);
  }

  Job job = Job.getInstance(getConf(),getConf().get("mapreduce.job.name",
      "syncTable_" + sourceTableName + "-" + targetTableName));
  Configuration jobConf = job.getConfiguration();
  if ("kerberos".equalsIgnoreCase(jobConf.get("hadoop.security.authentication"))) {
    TokenCache.obtainTokensForNamenodes(job.getCredentials(), new
        Path[] { sourceHashDir }, getConf());
  }

  HashTable.TableHash tableHash = HashTable.TableHash.read(getConf(), sourceHashDir);
  LOG.info("Read source hash manifest: " + tableHash);
  LOG.info("Read " + tableHash.partitions.size() + " partition keys");
  if (!tableHash.tableName.equals(sourceTableName)) {
    LOG.warn("Table name mismatch - manifest indicates hash was taken from: "
        + tableHash.tableName + " but job is reading from: " + sourceTableName);
  }
  if (tableHash.numHashFiles != tableHash.partitions.size() + 1) {
    throw new RuntimeException("Hash data appears corrupt. The number of of hash files created"
        + " should be 1 more than the number of partition keys.  However, the manifest file "
        + " says numHashFiles=" + tableHash.numHashFiles + " but the number of partition keys"
        + " found in the partitions file is " + tableHash.partitions.size());
  }

  Path dataDir = new Path(sourceHashDir, HashTable.HASH_DATA_DIR);
  int dataSubdirCount = 0;
  for (FileStatus file : fs.listStatus(dataDir)) {
    if (file.getPath().getName().startsWith(HashTable.OUTPUT_DATA_FILE_PREFIX)) {
      dataSubdirCount++;
    }
  }

  if (dataSubdirCount != tableHash.numHashFiles) {
    throw new RuntimeException("Hash data appears corrupt. The number of of hash files created"
        + " should be 1 more than the number of partition keys.  However, the number of data dirs"
        + " found is " + dataSubdirCount + " but the number of partition keys"
        + " found in the partitions file is " + tableHash.partitions.size());
  }

  job.setJarByClass(HashTable.class);
  jobConf.set(SOURCE_HASH_DIR_CONF_KEY, sourceHashDir.toString());
  jobConf.set(SOURCE_TABLE_CONF_KEY, sourceTableName);
  jobConf.set(TARGET_TABLE_CONF_KEY, targetTableName);
  if (sourceZkCluster != null) {
    jobConf.set(SOURCE_ZK_CLUSTER_CONF_KEY, sourceZkCluster);
    initCredentialsForHBase(sourceZkCluster, job);
  }
  if (targetZkCluster != null) {
    jobConf.set(TARGET_ZK_CLUSTER_CONF_KEY, targetZkCluster);
    initCredentialsForHBase(targetZkCluster, job);
  }
  jobConf.setBoolean(DRY_RUN_CONF_KEY, dryRun);
  jobConf.setBoolean(DO_DELETES_CONF_KEY, doDeletes);
  jobConf.setBoolean(DO_PUTS_CONF_KEY, doPuts);
  jobConf.setBoolean(IGNORE_TIMESTAMPS, ignoreTimestamps);

  TableMapReduceUtil.initTableMapperJob(targetTableName, tableHash.initScan(),
      SyncMapper.class, null, null, job);

  job.setNumReduceTasks(0);

  if (dryRun) {
    job.setOutputFormatClass(NullOutputFormat.class);
  } else {
    // No reducers.  Just write straight to table.  Call initTableReducerJob
    // because it sets up the TableOutputFormat.
    TableMapReduceUtil.initTableReducerJob(targetTableName, null, job, null,
        targetZkCluster, null, null);

    // would be nice to add an option for bulk load instead
  }

  // Obtain an authentication token, for the specified cluster, on behalf of the current user
  if (sourceZkCluster != null) {
    Configuration peerConf =
        HBaseConfiguration.createClusterConf(job.getConfiguration(), sourceZkCluster);
    TableMapReduceUtil.initCredentialsForCluster(job, peerConf);
  }
  return job;
}
 
Example 10
Source File: ExportSnapshot.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * Run Map-Reduce Job to perform the files copy.
 */
private void runCopyJob(final Path inputRoot, final Path outputRoot,
    final String snapshotName, final Path snapshotDir, final boolean verifyChecksum,
    final String filesUser, final String filesGroup, final int filesMode,
    final int mappers, final int bandwidthMB)
        throws IOException, InterruptedException, ClassNotFoundException {
  Configuration conf = getConf();
  if (filesGroup != null) conf.set(CONF_FILES_GROUP, filesGroup);
  if (filesUser != null) conf.set(CONF_FILES_USER, filesUser);
  if (mappers > 0) {
    conf.setInt(CONF_NUM_SPLITS, mappers);
    conf.setInt(MR_NUM_MAPS, mappers);
  }
  conf.setInt(CONF_FILES_MODE, filesMode);
  conf.setBoolean(CONF_CHECKSUM_VERIFY, verifyChecksum);
  conf.set(CONF_OUTPUT_ROOT, outputRoot.toString());
  conf.set(CONF_INPUT_ROOT, inputRoot.toString());
  conf.setInt(CONF_BANDWIDTH_MB, bandwidthMB);
  conf.set(CONF_SNAPSHOT_NAME, snapshotName);
  conf.set(CONF_SNAPSHOT_DIR, snapshotDir.toString());

  String jobname = conf.get(CONF_MR_JOB_NAME, "ExportSnapshot-" + snapshotName);
  Job job = new Job(conf);
  job.setJobName(jobname);
  job.setJarByClass(ExportSnapshot.class);
  TableMapReduceUtil.addDependencyJars(job);
  job.setMapperClass(ExportMapper.class);
  job.setInputFormatClass(ExportSnapshotInputFormat.class);
  job.setOutputFormatClass(NullOutputFormat.class);
  job.setMapSpeculativeExecution(false);
  job.setNumReduceTasks(0);

  // Acquire the delegation Tokens
  Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX);
  TokenCache.obtainTokensForNamenodes(job.getCredentials(),
    new Path[] { inputRoot }, srcConf);
  Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX);
  TokenCache.obtainTokensForNamenodes(job.getCredentials(),
      new Path[] { outputRoot }, destConf);

  // Run the MR Job
  if (!job.waitForCompletion(true)) {
    throw new ExportSnapshotException(job.getStatus().getFailureInfo());
  }
}
 
Example 11
Source File: TableMapReduceUtil.java    From hbase with Apache License 2.0 3 votes vote down vote up
/**
 * Obtain an authentication token, for the specified cluster, on behalf of the current user
 * and add it to the credentials for the given map reduce job.
 *
 * The quorumAddress is the key to the ZK ensemble, which contains:
 * hbase.zookeeper.quorum, hbase.zookeeper.client.port and
 * zookeeper.znode.parent
 *
 * @param job The job that requires the permission.
 * @param quorumAddress string that contains the 3 required configuratins
 * @throws IOException When the authentication token cannot be obtained.
 * @deprecated Since 1.2.0 and will be removed in 3.0.0. Use
 *   {@link #initCredentialsForCluster(Job, Configuration)} instead.
 * @see #initCredentialsForCluster(Job, Configuration)
 * @see <a href="https://issues.apache.org/jira/browse/HBASE-14886">HBASE-14886</a>
 */
@Deprecated
public static void initCredentialsForCluster(Job job, String quorumAddress)
    throws IOException {
  Configuration peerConf = HBaseConfiguration.createClusterConf(job.getConfiguration(),
      quorumAddress);
  initCredentialsForCluster(job, peerConf);
}