Java Code Examples for org.apache.hadoop.hbase.HBaseConfiguration#createClusterConf()
The following examples show how to use
org.apache.hadoop.hbase.HBaseConfiguration#createClusterConf() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ReplicationUtils.java From hbase with Apache License 2.0 | 6 votes |
public static Configuration getPeerClusterConfiguration(ReplicationPeerConfig peerConfig, Configuration baseConf) throws ReplicationException { Configuration otherConf; try { otherConf = HBaseConfiguration.createClusterConf(baseConf, peerConfig.getClusterKey()); } catch (IOException e) { throw new ReplicationException("Can't get peer configuration for peer " + peerConfig, e); } if (!peerConfig.getConfiguration().isEmpty()) { CompoundConfiguration compound = new CompoundConfiguration(); compound.add(otherConf); compound.addStringMap(peerConfig.getConfiguration()); return compound; } return otherConf; }
Example 2
Source File: TableOutputFormat.java From hbase with Apache License 2.0 | 6 votes |
@Override public void setConf(Configuration otherConf) { String tableName = otherConf.get(OUTPUT_TABLE); if(tableName == null || tableName.length() <= 0) { throw new IllegalArgumentException("Must specify table name"); } String address = otherConf.get(QUORUM_ADDRESS); int zkClientPort = otherConf.getInt(QUORUM_PORT, 0); String serverClass = otherConf.get(REGION_SERVER_CLASS); String serverImpl = otherConf.get(REGION_SERVER_IMPL); try { this.conf = HBaseConfiguration.createClusterConf(otherConf, address, OUTPUT_CONF_PREFIX); if (serverClass != null) { this.conf.set(HConstants.REGION_SERVER_IMPL, serverImpl); } if (zkClientPort != 0) { this.conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort); } } catch(IOException e) { LOG.error(e.toString(), e); throw new RuntimeException(e); } }
Example 3
Source File: ReplicationPeerConfigUtil.java From hbase with Apache License 2.0 | 6 votes |
/** * Returns the configuration needed to talk to the remote slave cluster. * @param conf the base configuration * @param peer the description of replication peer * @return the configuration for the peer cluster, null if it was unable to get the configuration * @throws IOException when create peer cluster configuration failed */ public static Configuration getPeerClusterConfiguration(Configuration conf, ReplicationPeerDescription peer) throws IOException { ReplicationPeerConfig peerConfig = peer.getPeerConfig(); Configuration otherConf; try { otherConf = HBaseConfiguration.createClusterConf(conf, peerConfig.getClusterKey()); } catch (IOException e) { throw new IOException("Can't get peer configuration for peerId=" + peer.getPeerId(), e); } if (!peerConfig.getConfiguration().isEmpty()) { CompoundConfiguration compound = new CompoundConfiguration(); compound.add(otherConf); compound.addStringMap(peerConfig.getConfiguration()); return compound; } return otherConf; }
Example 4
Source File: VerifyReplication.java From hbase with Apache License 2.0 | 5 votes |
private void restoreSnapshotForPeerCluster(Configuration conf, String peerQuorumAddress) throws IOException { Configuration peerConf = HBaseConfiguration.createClusterConf(conf, peerQuorumAddress, PEER_CONFIG_PREFIX); FileSystem.setDefaultUri(peerConf, peerFSAddress); CommonFSUtils.setRootDir(peerConf, new Path(peerFSAddress, peerHBaseRootAddress)); FileSystem fs = FileSystem.get(peerConf); RestoreSnapshotHelper.copySnapshotForScanner(peerConf, fs, CommonFSUtils.getRootDir(peerConf), new Path(peerFSAddress, peerSnapshotTmpDir), peerSnapshotName); }
Example 5
Source File: SyncTable.java From hbase with Apache License 2.0 | 5 votes |
private void initCredentialsForHBase(String zookeeper, Job job) throws IOException { Configuration peerConf = HBaseConfiguration.createClusterConf(job .getConfiguration(), zookeeper); if("kerberos".equalsIgnoreCase(peerConf.get("hbase.security.authentication"))){ TableMapReduceUtil.initCredentialsForCluster(job, peerConf); } }
Example 6
Source File: SyncTable.java From hbase with Apache License 2.0 | 5 votes |
private static Connection openConnection(Configuration conf, String zkClusterConfKey, String configPrefix) throws IOException { String zkCluster = conf.get(zkClusterConfKey); Configuration clusterConf = HBaseConfiguration.createClusterConf(conf, zkCluster, configPrefix); return ConnectionFactory.createConnection(clusterConf); }
Example 7
Source File: TableMapReduceUtil.java From hbase with Apache License 2.0 | 5 votes |
public static void initCredentials(Job job) throws IOException { UserProvider userProvider = UserProvider.instantiate(job.getConfiguration()); if (userProvider.isHadoopSecurityEnabled()) { // propagate delegation related props from launcher job to MR job if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) { job.getConfiguration().set("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION")); } } if (userProvider.isHBaseSecurityEnabled()) { try { // init credentials for remote cluster String quorumAddress = job.getConfiguration().get(TableOutputFormat.QUORUM_ADDRESS); User user = userProvider.getCurrent(); if (quorumAddress != null) { Configuration peerConf = HBaseConfiguration.createClusterConf(job.getConfiguration(), quorumAddress, TableOutputFormat.OUTPUT_CONF_PREFIX); Connection peerConn = ConnectionFactory.createConnection(peerConf); try { TokenUtil.addTokenForJob(peerConn, user, job); } finally { peerConn.close(); } } Connection conn = ConnectionFactory.createConnection(job.getConfiguration()); try { TokenUtil.addTokenForJob(conn, user, job); } finally { conn.close(); } } catch (InterruptedException ie) { LOG.info("Interrupted obtaining user authentication token"); Thread.currentThread().interrupt(); } } }
Example 8
Source File: TestZKConfig.java From hbase with Apache License 2.0 | 5 votes |
private void testKey(String ensemble, int port, String znode, Boolean multiplePortSupport) throws IOException { Configuration conf = new Configuration(); String key = ensemble+":"+port+":"+znode; String ensemble2 = null; ZKConfig.ZKClusterKey zkClusterKey = ZKConfig.transformClusterKey(key); if (multiplePortSupport) { ensemble2 = ZKConfig.standardizeZKQuorumServerString(ensemble, Integer.toString(port)); assertEquals(ensemble2, zkClusterKey.getQuorumString()); } else { assertEquals(ensemble, zkClusterKey.getQuorumString()); } assertEquals(port, zkClusterKey.getClientPort()); assertEquals(znode, zkClusterKey.getZnodeParent()); conf = HBaseConfiguration.createClusterConf(conf, key); assertEquals(zkClusterKey.getQuorumString(), conf.get(HConstants.ZOOKEEPER_QUORUM)); assertEquals(zkClusterKey.getClientPort(), conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT, -1)); assertEquals(zkClusterKey.getZnodeParent(), conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT)); String reconstructedKey = ZKConfig.getZooKeeperClusterKey(conf); if (multiplePortSupport) { String key2 = ensemble2 + ":" + port + ":" + znode; assertEquals(key2, reconstructedKey); } else { assertEquals(key, reconstructedKey); } }
Example 9
Source File: SyncTable.java From hbase with Apache License 2.0 | 4 votes |
public Job createSubmittableJob(String[] args) throws IOException { FileSystem fs = sourceHashDir.getFileSystem(getConf()); if (!fs.exists(sourceHashDir)) { throw new IOException("Source hash dir not found: " + sourceHashDir); } Job job = Job.getInstance(getConf(),getConf().get("mapreduce.job.name", "syncTable_" + sourceTableName + "-" + targetTableName)); Configuration jobConf = job.getConfiguration(); if ("kerberos".equalsIgnoreCase(jobConf.get("hadoop.security.authentication"))) { TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { sourceHashDir }, getConf()); } HashTable.TableHash tableHash = HashTable.TableHash.read(getConf(), sourceHashDir); LOG.info("Read source hash manifest: " + tableHash); LOG.info("Read " + tableHash.partitions.size() + " partition keys"); if (!tableHash.tableName.equals(sourceTableName)) { LOG.warn("Table name mismatch - manifest indicates hash was taken from: " + tableHash.tableName + " but job is reading from: " + sourceTableName); } if (tableHash.numHashFiles != tableHash.partitions.size() + 1) { throw new RuntimeException("Hash data appears corrupt. The number of of hash files created" + " should be 1 more than the number of partition keys. However, the manifest file " + " says numHashFiles=" + tableHash.numHashFiles + " but the number of partition keys" + " found in the partitions file is " + tableHash.partitions.size()); } Path dataDir = new Path(sourceHashDir, HashTable.HASH_DATA_DIR); int dataSubdirCount = 0; for (FileStatus file : fs.listStatus(dataDir)) { if (file.getPath().getName().startsWith(HashTable.OUTPUT_DATA_FILE_PREFIX)) { dataSubdirCount++; } } if (dataSubdirCount != tableHash.numHashFiles) { throw new RuntimeException("Hash data appears corrupt. The number of of hash files created" + " should be 1 more than the number of partition keys. However, the number of data dirs" + " found is " + dataSubdirCount + " but the number of partition keys" + " found in the partitions file is " + tableHash.partitions.size()); } job.setJarByClass(HashTable.class); jobConf.set(SOURCE_HASH_DIR_CONF_KEY, sourceHashDir.toString()); jobConf.set(SOURCE_TABLE_CONF_KEY, sourceTableName); jobConf.set(TARGET_TABLE_CONF_KEY, targetTableName); if (sourceZkCluster != null) { jobConf.set(SOURCE_ZK_CLUSTER_CONF_KEY, sourceZkCluster); initCredentialsForHBase(sourceZkCluster, job); } if (targetZkCluster != null) { jobConf.set(TARGET_ZK_CLUSTER_CONF_KEY, targetZkCluster); initCredentialsForHBase(targetZkCluster, job); } jobConf.setBoolean(DRY_RUN_CONF_KEY, dryRun); jobConf.setBoolean(DO_DELETES_CONF_KEY, doDeletes); jobConf.setBoolean(DO_PUTS_CONF_KEY, doPuts); jobConf.setBoolean(IGNORE_TIMESTAMPS, ignoreTimestamps); TableMapReduceUtil.initTableMapperJob(targetTableName, tableHash.initScan(), SyncMapper.class, null, null, job); job.setNumReduceTasks(0); if (dryRun) { job.setOutputFormatClass(NullOutputFormat.class); } else { // No reducers. Just write straight to table. Call initTableReducerJob // because it sets up the TableOutputFormat. TableMapReduceUtil.initTableReducerJob(targetTableName, null, job, null, targetZkCluster, null, null); // would be nice to add an option for bulk load instead } // Obtain an authentication token, for the specified cluster, on behalf of the current user if (sourceZkCluster != null) { Configuration peerConf = HBaseConfiguration.createClusterConf(job.getConfiguration(), sourceZkCluster); TableMapReduceUtil.initCredentialsForCluster(job, peerConf); } return job; }
Example 10
Source File: ExportSnapshot.java From hbase with Apache License 2.0 | 4 votes |
/** * Run Map-Reduce Job to perform the files copy. */ private void runCopyJob(final Path inputRoot, final Path outputRoot, final String snapshotName, final Path snapshotDir, final boolean verifyChecksum, final String filesUser, final String filesGroup, final int filesMode, final int mappers, final int bandwidthMB) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = getConf(); if (filesGroup != null) conf.set(CONF_FILES_GROUP, filesGroup); if (filesUser != null) conf.set(CONF_FILES_USER, filesUser); if (mappers > 0) { conf.setInt(CONF_NUM_SPLITS, mappers); conf.setInt(MR_NUM_MAPS, mappers); } conf.setInt(CONF_FILES_MODE, filesMode); conf.setBoolean(CONF_CHECKSUM_VERIFY, verifyChecksum); conf.set(CONF_OUTPUT_ROOT, outputRoot.toString()); conf.set(CONF_INPUT_ROOT, inputRoot.toString()); conf.setInt(CONF_BANDWIDTH_MB, bandwidthMB); conf.set(CONF_SNAPSHOT_NAME, snapshotName); conf.set(CONF_SNAPSHOT_DIR, snapshotDir.toString()); String jobname = conf.get(CONF_MR_JOB_NAME, "ExportSnapshot-" + snapshotName); Job job = new Job(conf); job.setJobName(jobname); job.setJarByClass(ExportSnapshot.class); TableMapReduceUtil.addDependencyJars(job); job.setMapperClass(ExportMapper.class); job.setInputFormatClass(ExportSnapshotInputFormat.class); job.setOutputFormatClass(NullOutputFormat.class); job.setMapSpeculativeExecution(false); job.setNumReduceTasks(0); // Acquire the delegation Tokens Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX); TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { inputRoot }, srcConf); Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX); TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { outputRoot }, destConf); // Run the MR Job if (!job.waitForCompletion(true)) { throw new ExportSnapshotException(job.getStatus().getFailureInfo()); } }
Example 11
Source File: TableMapReduceUtil.java From hbase with Apache License 2.0 | 3 votes |
/** * Obtain an authentication token, for the specified cluster, on behalf of the current user * and add it to the credentials for the given map reduce job. * * The quorumAddress is the key to the ZK ensemble, which contains: * hbase.zookeeper.quorum, hbase.zookeeper.client.port and * zookeeper.znode.parent * * @param job The job that requires the permission. * @param quorumAddress string that contains the 3 required configuratins * @throws IOException When the authentication token cannot be obtained. * @deprecated Since 1.2.0 and will be removed in 3.0.0. Use * {@link #initCredentialsForCluster(Job, Configuration)} instead. * @see #initCredentialsForCluster(Job, Configuration) * @see <a href="https://issues.apache.org/jira/browse/HBASE-14886">HBASE-14886</a> */ @Deprecated public static void initCredentialsForCluster(Job job, String quorumAddress) throws IOException { Configuration peerConf = HBaseConfiguration.createClusterConf(job.getConfiguration(), quorumAddress); initCredentialsForCluster(job, peerConf); }