Java Code Examples for org.apache.hadoop.hbase.CellUtil#getCellKeyAsString()

The following examples show how to use org.apache.hadoop.hbase.CellUtil#getCellKeyAsString() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SplitRegionScanner.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up
@Override
public void close() throws IOException {
    String lastRow = lastCell != null ? CellUtil.getCellKeyAsString(lastCell) : null;
    if (LOG.isDebugEnabled())
        LOG.debug(String.format("close split scanner with table [%s], scan [%s] with rowCount=%d, reinitCount=%d, scannerExceptionCount=%d, lastRows=%s",htable.getName().toString(),initialScan,totalScannerCount,reInitCount,scanExceptionCount,lastRow));
    closed = true;
    for (RegionScanner rs : regionScanners) {
        rs.close();
    }
    regionScanners.clear();
    clientPartition.close();
    currentScanner = null;

}
 
Example 2
Source File: HStore.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * This throws a WrongRegionException if the HFile does not fit in this region, or an
 * InvalidHFileException if the HFile is not valid.
 */
public void assertBulkLoadHFileOk(Path srcPath) throws IOException {
  HFile.Reader reader  = null;
  try {
    LOG.info("Validating hfile at " + srcPath + " for inclusion in " + this);
    FileSystem srcFs = srcPath.getFileSystem(conf);
    srcFs.access(srcPath, FsAction.READ_WRITE);
    reader = HFile.createReader(srcFs, srcPath, cacheConf, isPrimaryReplicaStore(), conf);

    Optional<byte[]> firstKey = reader.getFirstRowKey();
    Preconditions.checkState(firstKey.isPresent(), "First key can not be null");
    Optional<Cell> lk = reader.getLastKey();
    Preconditions.checkState(lk.isPresent(), "Last key can not be null");
    byte[] lastKey =  CellUtil.cloneRow(lk.get());

    if (LOG.isDebugEnabled()) {
      LOG.debug("HFile bounds: first=" + Bytes.toStringBinary(firstKey.get()) +
          " last=" + Bytes.toStringBinary(lastKey));
      LOG.debug("Region bounds: first=" +
          Bytes.toStringBinary(getRegionInfo().getStartKey()) +
          " last=" + Bytes.toStringBinary(getRegionInfo().getEndKey()));
    }

    if (!this.getRegionInfo().containsRange(firstKey.get(), lastKey)) {
      throw new WrongRegionException(
          "Bulk load file " + srcPath.toString() + " does not fit inside region "
          + this.getRegionInfo().getRegionNameAsString());
    }

    if(reader.length() > conf.getLong(HConstants.HREGION_MAX_FILESIZE,
        HConstants.DEFAULT_MAX_FILE_SIZE)) {
      LOG.warn("Trying to bulk load hfile " + srcPath + " with size: " +
          reader.length() + " bytes can be problematic as it may lead to oversplitting.");
    }

    if (verifyBulkLoads) {
      long verificationStartTime = EnvironmentEdgeManager.currentTime();
      LOG.info("Full verification started for bulk load hfile: {}", srcPath);
      Cell prevCell = null;
      HFileScanner scanner = reader.getScanner(false, false, false);
      scanner.seekTo();
      do {
        Cell cell = scanner.getCell();
        if (prevCell != null) {
          if (comparator.compareRows(prevCell, cell) > 0) {
            throw new InvalidHFileException("Previous row is greater than"
                + " current row: path=" + srcPath + " previous="
                + CellUtil.getCellKeyAsString(prevCell) + " current="
                + CellUtil.getCellKeyAsString(cell));
          }
          if (CellComparator.getInstance().compareFamilies(prevCell, cell) != 0) {
            throw new InvalidHFileException("Previous key had different"
                + " family compared to current key: path=" + srcPath
                + " previous="
                + Bytes.toStringBinary(prevCell.getFamilyArray(), prevCell.getFamilyOffset(),
                    prevCell.getFamilyLength())
                + " current="
                + Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(),
                    cell.getFamilyLength()));
          }
        }
        prevCell = cell;
      } while (scanner.next());
      LOG.info("Full verification complete for bulk load hfile: " + srcPath.toString() +
        " took " + (EnvironmentEdgeManager.currentTime() - verificationStartTime) + " ms");
    }
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
}
 
Example 3
Source File: HFileWriterImpl.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * Try to return a Cell that falls between <code>left</code> and
 * <code>right</code> but that is shorter; i.e. takes up less space. This
 * trick is used building HFile block index. Its an optimization. It does not
 * always work. In this case we'll just return the <code>right</code> cell.
 * @return A cell that sorts between <code>left</code> and <code>right</code>.
 */
public static Cell getMidpoint(final CellComparator comparator, final Cell left,
    final Cell right) {
  // TODO: Redo so only a single pass over the arrays rather than one to
  // compare and then a second composing midpoint.
  if (right == null) {
    throw new IllegalArgumentException("right cell can not be null");
  }
  if (left == null) {
    return right;
  }
  // If Cells from meta table, don't mess around. meta table Cells have schema
  // (table,startrow,hash) so can't be treated as plain byte arrays. Just skip
  // out without trying to do this optimization.
  if (comparator instanceof MetaCellComparator) {
    return right;
  }
  int diff = comparator.compareRows(left, right);
  if (diff > 0) {
    throw new IllegalArgumentException("Left row sorts after right row; left="
        + CellUtil.getCellKeyAsString(left) + ", right=" + CellUtil.getCellKeyAsString(right));
  }
  byte[] midRow;
  boolean bufferBacked = left instanceof ByteBufferExtendedCell
      && right instanceof ByteBufferExtendedCell;
  if (diff < 0) {
    // Left row is < right row.
    if (bufferBacked) {
      midRow = getMinimumMidpointArray(((ByteBufferExtendedCell) left).getRowByteBuffer(),
          ((ByteBufferExtendedCell) left).getRowPosition(), left.getRowLength(),
          ((ByteBufferExtendedCell) right).getRowByteBuffer(),
          ((ByteBufferExtendedCell) right).getRowPosition(), right.getRowLength());
    } else {
      midRow = getMinimumMidpointArray(left.getRowArray(), left.getRowOffset(),
          left.getRowLength(), right.getRowArray(), right.getRowOffset(), right.getRowLength());
    }
    // If midRow is null, just return 'right'. Can't do optimization.
    if (midRow == null) {
      return right;
    }
    return PrivateCellUtil.createFirstOnRow(midRow);
  }
  // Rows are same. Compare on families.
  diff = comparator.compareFamilies(left, right);
  if (diff > 0) {
    throw new IllegalArgumentException("Left family sorts after right family; left="
        + CellUtil.getCellKeyAsString(left) + ", right=" + CellUtil.getCellKeyAsString(right));
  }
  if (diff < 0) {
    if (bufferBacked) {
      midRow = getMinimumMidpointArray(((ByteBufferExtendedCell) left).getFamilyByteBuffer(),
          ((ByteBufferExtendedCell) left).getFamilyPosition(), left.getFamilyLength(),
          ((ByteBufferExtendedCell) right).getFamilyByteBuffer(),
          ((ByteBufferExtendedCell) right).getFamilyPosition(), right.getFamilyLength());
    } else {
      midRow = getMinimumMidpointArray(left.getFamilyArray(), left.getFamilyOffset(),
          left.getFamilyLength(), right.getFamilyArray(), right.getFamilyOffset(),
          right.getFamilyLength());
    }
    // If midRow is null, just return 'right'. Can't do optimization.
    if (midRow == null) {
      return right;
    }
    // Return new Cell where we use right row and then a mid sort family.
    return PrivateCellUtil.createFirstOnRowFamily(right, midRow, 0, midRow.length);
  }
  // Families are same. Compare on qualifiers.
  diff = comparator.compareQualifiers(left, right);
  if (diff > 0) {
    throw new IllegalArgumentException("Left qualifier sorts after right qualifier; left="
        + CellUtil.getCellKeyAsString(left) + ", right=" + CellUtil.getCellKeyAsString(right));
  }
  if (diff < 0) {
    if (bufferBacked) {
      midRow = getMinimumMidpointArray(((ByteBufferExtendedCell) left).getQualifierByteBuffer(),
          ((ByteBufferExtendedCell) left).getQualifierPosition(), left.getQualifierLength(),
          ((ByteBufferExtendedCell) right).getQualifierByteBuffer(),
          ((ByteBufferExtendedCell) right).getQualifierPosition(), right.getQualifierLength());
    } else {
      midRow = getMinimumMidpointArray(left.getQualifierArray(), left.getQualifierOffset(),
          left.getQualifierLength(), right.getQualifierArray(), right.getQualifierOffset(),
          right.getQualifierLength());
    }
    // If midRow is null, just return 'right'. Can't do optimization.
    if (midRow == null) {
      return right;
    }
    // Return new Cell where we use right row and family and then a mid sort qualifier.
    return PrivateCellUtil.createFirstOnRowCol(right, midRow, 0, midRow.length);
  }
  // No opportunity for optimization. Just return right key.
  return right;
}
 
Example 4
Source File: HFileReaderImpl.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * Within a loaded block, seek looking for the last key that is smaller than
 * (or equal to?) the key we are interested in.
 * A note on the seekBefore: if you have seekBefore = true, AND the first
 * key in the block = key, then you'll get thrown exceptions. The caller has
 * to check for that case and load the previous block as appropriate.
 * @param key
 *          the key to find
 * @param seekBefore
 *          find the key before the given key in case of exact match.
 * @return 0 in case of an exact key match, 1 in case of an inexact match,
 *         -2 in case of an inexact match and furthermore, the input key
 *         less than the first key of current block(e.g. using a faked index
 *         key)
 */
protected int blockSeek(Cell key, boolean seekBefore) {
  int klen, vlen, tlen = 0;
  int lastKeyValueSize = -1;
  int offsetFromPos;
  do {
    offsetFromPos = 0;
    // Better to ensure that we use the BB Utils here
    long ll = blockBuffer.getLongAfterPosition(offsetFromPos);
    klen = (int)(ll >> Integer.SIZE);
    vlen = (int)(Bytes.MASK_FOR_LOWER_INT_IN_LONG ^ ll);
    if (checkKeyLen(klen) || checkLen(vlen)) {
      throw new IllegalStateException("Invalid klen " + klen + " or vlen "
          + vlen + ". Block offset: "
          + curBlock.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
          + blockBuffer.position() + " (without header)."
          + " path=" + reader.getPath());
    }
    offsetFromPos += Bytes.SIZEOF_LONG;
    blockBuffer.asSubByteBuffer(blockBuffer.position() + offsetFromPos, klen, pair);
    bufBackedKeyOnlyKv.setKey(pair.getFirst(), pair.getSecond(), klen);
    int comp =
        PrivateCellUtil.compareKeyIgnoresMvcc(reader.getComparator(), key, bufBackedKeyOnlyKv);
    offsetFromPos += klen + vlen;
    if (this.reader.getFileContext().isIncludesTags()) {
      // Read short as unsigned, high byte first
      tlen = ((blockBuffer.getByteAfterPosition(offsetFromPos) & 0xff) << 8)
          ^ (blockBuffer.getByteAfterPosition(offsetFromPos + 1) & 0xff);
      if (checkLen(tlen)) {
        throw new IllegalStateException("Invalid tlen " + tlen + ". Block offset: "
            + curBlock.getOffset() + ", block length: " + blockBuffer.limit() + ", position: "
            + blockBuffer.position() + " (without header)."
            + " path=" + reader.getPath());
      }
      // add the two bytes read for the tags.
      offsetFromPos += tlen + (Bytes.SIZEOF_SHORT);
    }
    if (this.reader.getHFileInfo().shouldIncludeMemStoreTS()) {
      // Directly read the mvcc based on current position
      readMvccVersion(offsetFromPos);
    }
    if (comp == 0) {
      if (seekBefore) {
        if (lastKeyValueSize < 0) {
          throw new IllegalStateException("blockSeek with seekBefore "
              + "at the first key of the block: key=" + CellUtil.getCellKeyAsString(key)
              + ", blockOffset=" + curBlock.getOffset() + ", onDiskSize="
              + curBlock.getOnDiskSizeWithHeader()
              + ", path=" + reader.getPath());
        }
        blockBuffer.moveBack(lastKeyValueSize);
        readKeyValueLen();
        return 1; // non exact match.
      }
      currKeyLen = klen;
      currValueLen = vlen;
      currTagsLen = tlen;
      return 0; // indicate exact match
    } else if (comp < 0) {
      if (lastKeyValueSize > 0) {
        blockBuffer.moveBack(lastKeyValueSize);
      }
      readKeyValueLen();
      if (lastKeyValueSize == -1 && blockBuffer.position() == 0) {
        return HConstants.INDEX_KEY_MAGIC;
      }
      return 1;
    }
    // The size of this key/value tuple, including key/value length fields.
    lastKeyValueSize = klen + vlen + currMemstoreTSLen + KEY_VALUE_LEN_SIZE;
    // include tag length also if tags included with KV
    if (reader.getFileContext().isIncludesTags()) {
      lastKeyValueSize += tlen + Bytes.SIZEOF_SHORT;
    }
    blockBuffer.skip(lastKeyValueSize);
  } while (blockBuffer.hasRemaining());

  // Seek to the last key we successfully read. This will happen if this is
  // the last key/value pair in the file, in which case the following call
  // to next() has to return false.
  blockBuffer.moveBack(lastKeyValueSize);
  readKeyValueLen();
  return 1; // didn't exactly find it.
}
 
Example 5
Source File: RowIndexSeekerV1.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public String toString() {
  return CellUtil.getCellKeyAsString(toCell());
}