Java Code Examples for org.apache.hadoop.hbase.client.RegionLocator#getStartKeys()

/**
 * Return the start keys of all of the regions in this table,
 * as a list of ImmutableBytesWritable.
 */
private static List<ImmutableBytesWritable> getRegionStartKeys(List<RegionLocator> regionLocators,
                                                               boolean writeMultipleTables)
        throws IOException {

  ArrayList<ImmutableBytesWritable> ret = new ArrayList<>();
  for(RegionLocator regionLocator : regionLocators) {
    TableName tableName = regionLocator.getName();
    LOG.info("Looking up current regions for table " + tableName);
    byte[][] byteKeys = regionLocator.getStartKeys();
    for (byte[] byteKey : byteKeys) {
      byte[] fullKey = byteKey; //HFileOutputFormat2 use case
      if (writeMultipleTables) {
        //MultiTableHFileOutputFormat use case
        fullKey = combineTableNameSuffix(tableName.getName(), byteKey);
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug("SplitPoint startkey for " + tableName + ": " + Bytes.toStringBinary(fullKey));
      }
      ret.add(new ImmutableBytesWritable(fullKey));
    }
  }
  return ret;
}
 

/**
 * Return the start keys of all of the regions in this table,
 * as a list of ImmutableBytesWritable.
 */
private static List<ImmutableBytesWritable> getRegionStartKeys(RegionLocator table) throws IOException {
    byte[][] byteKeys = table.getStartKeys();
    ArrayList<ImmutableBytesWritable> ret = new ArrayList<ImmutableBytesWritable>(byteKeys.length);
    for (byte[] byteKey : byteKeys) {
        ret.add(new ImmutableBytesWritable(byteKey));
    }
    return ret;
}
 

/**
 * Return the start keys of all of the regions in this table,
 * as a list of ImmutableBytesWritable.
 */
private static List<ImmutableBytesWritable> getRegionStartKeys(RegionLocator table) throws IOException {
    byte[][] byteKeys = table.getStartKeys();
    ArrayList<ImmutableBytesWritable> ret = new ArrayList<ImmutableBytesWritable>(byteKeys.length);
    for (byte[] byteKey : byteKeys) {
        ret.add(new ImmutableBytesWritable(byteKey));
    }
    return ret;
}
 

private void setupIndexAndDataTable(Connection connection) throws SQLException, IOException {
    pDataTable = PhoenixRuntime.getTableNoCache(connection, qDataTable);
    if (!isValidIndexTable(connection, qDataTable, indexTable, tenantId)) {
        throw new IllegalArgumentException(
                String.format(" %s is not an index table for %s for this connection",
                        indexTable, qDataTable));
    }
    pIndexTable = PhoenixRuntime.getTable(connection, schemaName != null && !schemaName.isEmpty()
            ? SchemaUtil.getQualifiedTableName(schemaName, indexTable) : indexTable);
    indexType = pIndexTable.getIndexType();
    if (schemaName != null && !schemaName.isEmpty()) {
        qIndexTable = SchemaUtil.getQualifiedTableName(schemaName, indexTable);
    } else {
        qIndexTable = indexTable;
    }
    if (IndexType.LOCAL.equals(indexType)) {
        isLocalIndexBuild = true;
        try (org.apache.hadoop.hbase.client.Connection hConn
                = getTemporaryHConnection(connection.unwrap(PhoenixConnection.class))) {
            RegionLocator regionLocator = hConn
                    .getRegionLocator(TableName.valueOf(pIndexTable.getPhysicalName().getBytes()));
            splitKeysBeforeJob = regionLocator.getStartKeys();
        }
    }
    // We have to mark Disable index to Building before we can set it to Active in the reducer. Otherwise it errors out with
    // index state transition error
    changeDisabledIndexStateToBuiding(connection);
}
 

/**
 * Return the start keys of all of the regions in this table,
 * as a list of ImmutableBytesWritable.
 */
private static Set<TableRowkeyPair> getRegionStartKeys(String tableName , RegionLocator table) throws IOException {
  byte[][] byteKeys = table.getStartKeys();
  Set<TableRowkeyPair> ret = new TreeSet<TableRowkeyPair>();
  for (byte[] byteKey : byteKeys) {
      // phoenix-2216: start : passing the table name and startkey  
    ret.add(new TableRowkeyPair(tableName, new ImmutableBytesWritable(byteKey)));
  }
  return ret;
}
 

/**
 * Submits the jobs to the cluster.
 * Loads the HFiles onto the respective tables.
 * @throws Exception 
 */
public int submitJob(final Configuration conf, final String qualifiedTableName,
    final String inputPaths, final Path outputPath, List<TargetTableRef> tablesToBeLoaded, boolean hasLocalIndexes) throws Exception {
   
    Job job = Job.getInstance(conf, "Phoenix MapReduce import for " + qualifiedTableName);
    FileInputFormat.addInputPaths(job, inputPaths);
    FileOutputFormat.setOutputPath(job, outputPath);

    job.setInputFormatClass(PhoenixTextInputFormat.class);
    job.setMapOutputKeyClass(TableRowkeyPair.class);
    job.setMapOutputValueClass(ImmutableBytesWritable.class);
    job.setOutputKeyClass(TableRowkeyPair.class);
    job.setOutputValueClass(KeyValue.class);
    job.setReducerClass(FormatToKeyValueReducer.class);
    byte[][] splitKeysBeforeJob = null;
    try(org.apache.hadoop.hbase.client.Connection hbaseConn =
            ConnectionFactory.createConnection(job.getConfiguration())) {
        RegionLocator regionLocator = null;
        if(hasLocalIndexes) {
            try{
                regionLocator = hbaseConn.getRegionLocator(
                        TableName.valueOf(qualifiedTableName));
                splitKeysBeforeJob = regionLocator.getStartKeys();
            } finally {
                if (regionLocator != null) regionLocator.close();
            }
        }
        MultiHfileOutputFormat.configureIncrementalLoad(job, tablesToBeLoaded);

        final String tableNamesAsJson = TargetTableRefFunctions.NAMES_TO_JSON
                .apply(tablesToBeLoaded);
        final String logicalNamesAsJson = TargetTableRefFunctions.LOGICAL_NAMES_TO_JSON
                .apply(tablesToBeLoaded);

        job.getConfiguration().set(FormatToBytesWritableMapper.TABLE_NAMES_CONFKEY,
                tableNamesAsJson);
        job.getConfiguration().set(FormatToBytesWritableMapper.LOGICAL_NAMES_CONFKEY,
                logicalNamesAsJson);

        // give subclasses their hook
        setupJob(job);

        LOGGER.info("Running MapReduce import job from {} to {}", inputPaths, outputPath);
        boolean success = job.waitForCompletion(true);

        if (success) {
            if (hasLocalIndexes) {
                try {
                    regionLocator = hbaseConn.getRegionLocator(
                            TableName.valueOf(qualifiedTableName));
                    if(!IndexUtil.matchingSplitKeys(splitKeysBeforeJob,
                            regionLocator.getStartKeys())) {
                        LOGGER.error("The table " + qualifiedTableName + " has local indexes and"
                                + " there is split key mismatch before and after running"
                                + " bulkload job. Please rerun the job otherwise there may be"
                                + " inconsistencies between actual data and index data.");
                        return -1;
                    }
                } finally {
                    if (regionLocator != null) regionLocator.close();
                }
            }
            LOGGER.info("Loading HFiles from {}", outputPath);
            completebulkload(conf,outputPath,tablesToBeLoaded);
            LOGGER.info("Removing output directory {}", outputPath);
            if(!outputPath.getFileSystem(conf).delete(outputPath, true)) {
                LOGGER.error("Failed to delete the output directory {}", outputPath);
            }
            return 0;
        } else {
           return -1;
       }
   }
}