Java Code Examples for org.apache.hadoop.hbase.CellUtil#isDelete()

The following examples show how to use org.apache.hadoop.hbase.CellUtil#isDelete() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestKeepDeletes.java    From hbase with Apache License 2.0 6 votes vote down vote up
private int countDeleteMarkers(HRegion region) throws IOException {
  Scan s = new Scan();
  s.setRaw(true);
  // use max versions from the store(s)
  s.readVersions(region.getStores().iterator().next().getScanInfo().getMaxVersions());
  InternalScanner scan = region.getScanner(s);
  List<Cell> kvs = new ArrayList<>();
  int res = 0;
  boolean hasMore;
  do {
    hasMore = scan.next(kvs);
    for (Cell kv : kvs) {
      if(CellUtil.isDelete(kv)) {
        res++;
      }
    }
    kvs.clear();
  } while (hasMore);
  scan.close();
  return res;
}
 
Example 2
Source File: SepEventRowData.java    From hbase-indexer with Apache License 2.0 6 votes vote down vote up
/**
 * Makes a HBase Result object based on the KeyValue's from the SEP event. Usually, this will only be used in
 * situations where only new data is written (or updates are complete row updates), so we don't expect any
 * delete-type key-values, but just to be sure we filter them out.
 */
@Override
public Result toResult() {

    List<Cell> filteredKeyValues = Lists.newArrayListWithCapacity(sepEvent.getKeyValues().size());

    for (Cell kv : getKeyValues()) {
        if (!CellUtil.isDelete(kv) && !CellUtil.isDeleteFamily(kv)) {
            filteredKeyValues.add(kv);
        }
    }

    // A Result object requires that the KeyValues are sorted (e.g., it does binary search on them)
    Collections.sort(filteredKeyValues, KeyValue.COMPARATOR);
    return Result.create(filteredKeyValues);
}
 
Example 3
Source File: Indexer.java    From hbase-indexer with Apache License 2.0 6 votes vote down vote up
/**
 * Calculate a map of Solr document ids to relevant RowData, only taking the most recent event for each document id..
 */
private Map<String, RowData> calculateUniqueEvents(List<RowData> rowDataList) {
    Map<String, RowData> idToEvent = Maps.newHashMap();
    for (RowData rowData : rowDataList) {
        // Check if the event contains changes to relevant key values
        boolean relevant = false;
        for (Cell kv : rowData.getKeyValues()) {
            if (mapper.isRelevantKV((KeyValue)kv) || CellUtil.isDelete(kv)) {
                relevant = true;
                break;
            }
        }

        if (!relevant) {
            continue;
        }
        if (uniqueKeyFormatter instanceof UniqueTableKeyFormatter) {
            idToEvent.put(((UniqueTableKeyFormatter) uniqueKeyFormatter).formatRow(rowData.getRow(),
                    rowData.getTable()), rowData);
        } else {
            idToEvent.put(uniqueKeyFormatter.formatRow(rowData.getRow()), rowData);
        }

    }
    return idToEvent;
}
 
Example 4
Source File: IndexHalfStoreFileReader.java    From phoenix with Apache License 2.0 5 votes vote down vote up
private boolean isSatisfiedMidKeyCondition(Cell kv) {
    if (CellUtil.isDelete(kv) && kv.getValueLength() == 0) {
        // In case of a Delete type KV, let it be going to both the daughter regions.
        // No problems in doing so. In the correct daughter region where it belongs to, this delete
        // tomb will really delete a KV. In the other it will just hang around there with no actual
        // kv coming for which this is a delete tomb. :)
        return true;
    }
    ImmutableBytesWritable rowKey =
            new ImmutableBytesWritable(kv.getRowArray(), kv.getRowOffset() + offset,
                    kv.getRowLength() - offset);
    Entry<ImmutableBytesWritable, IndexMaintainer> entry = indexMaintainers.entrySet().iterator().next();
    IndexMaintainer indexMaintainer = entry.getValue();
    byte[] viewIndexId = indexMaintainer.getViewIndexIdFromIndexRowKey(rowKey);
    IndexMaintainer actualIndexMaintainer = indexMaintainers.get(new ImmutableBytesWritable(viewIndexId));
    byte[] dataRowKey = actualIndexMaintainer.buildDataRowKey(rowKey, this.viewConstants);
    int compareResult = Bytes.compareTo(dataRowKey, splitRow);
    if (top) {
        if (compareResult >= 0) {
            return true;
        }
    } else {
        if (compareResult < 0) {
            return true;
        }
    }
    return false;
}
 
Example 5
Source File: ProtobufUtil.java    From hbase with Apache License 2.0 5 votes vote down vote up
public static MutationProto toMutation(final MutationType type, final Mutation mutation,
    MutationProto.Builder builder, long nonce)
throws IOException {
  builder = getMutationBuilderAndSetCommonFields(type, mutation, builder);
  if (nonce != HConstants.NO_NONCE) {
    builder.setNonce(nonce);
  }
  if (type == MutationType.INCREMENT) {
    builder.setTimeRange(ProtobufUtil.toTimeRange(((Increment) mutation).getTimeRange()));
  }
  if (type == MutationType.APPEND) {
    builder.setTimeRange(ProtobufUtil.toTimeRange(((Append) mutation).getTimeRange()));
  }
  ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
  QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
  for (Map.Entry<byte[],List<Cell>> family: mutation.getFamilyCellMap().entrySet()) {
    columnBuilder.clear();
    columnBuilder.setFamily(UnsafeByteOperations.unsafeWrap(family.getKey()));
    for (Cell cell: family.getValue()) {
      valueBuilder.clear();
      valueBuilder.setQualifier(UnsafeByteOperations.unsafeWrap(
          cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
      valueBuilder.setValue(UnsafeByteOperations.unsafeWrap(
          cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
      valueBuilder.setTimestamp(cell.getTimestamp());
      if (type == MutationType.DELETE || (type == MutationType.PUT && CellUtil.isDelete(cell))) {
        KeyValue.Type keyValueType = KeyValue.Type.codeToType(cell.getTypeByte());
        valueBuilder.setDeleteType(toDeleteType(keyValueType));
      }
      columnBuilder.addQualifierValue(valueBuilder.build());
    }
    builder.addColumnValue(columnBuilder.build());
  }
  return builder.build();
}
 
Example 6
Source File: Indexer.java    From hbase-indexer with Apache License 2.0 5 votes vote down vote up
@Override
protected void calculateIndexUpdates(List<RowData> rowDataList, SolrUpdateCollector updateCollector) throws IOException {
    Map<String, KeyValue> idToKeyValue = calculateUniqueEvents(rowDataList);
    for (Entry<String, KeyValue> idToKvEntry : idToKeyValue.entrySet()) {
        String documentId = idToKvEntry.getKey();

        KeyValue keyValue = idToKvEntry.getValue();
        if (CellUtil.isDelete(keyValue)) {
            handleDelete(documentId, keyValue, updateCollector, uniqueKeyFormatter);
        } else {
            Result result = Result.create(Collections.<Cell>singletonList(keyValue));
            SolrUpdateWriter updateWriter = new RowAndFamilyAddingSolrUpdateWriter(
                    conf.getRowField(),
                    conf.getColumnFamilyField(),
                    uniqueKeyFormatter,
                    keyValue,
                    new IdAddingSolrUpdateWriter(
                            conf.getUniqueKeyField(),
                            documentId,
                            conf.getTableNameField(),
                            tableName,
                            updateCollector));

            mapper.map(result, updateWriter);

        }
    }
}
 
Example 7
Source File: CompactorScanner.java    From phoenix-omid with Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
public boolean shouldRetainNonTransactionallyDeletedCell(Cell cell) {
    return (CellUtil.isDelete(cell) || CellUtil.isDeleteFamily(cell))
            &&
            retainNonTransactionallyDeletedCells;
}
 
Example 8
Source File: TestCompaction.java    From phoenix-omid with Apache License 2.0 4 votes vote down vote up
@Test(timeOut = 60_000)
public void testACellDeletedNonTransactionallyIsPreservedWhenMinorCompactionOccurs() throws Throwable {
    String TEST_TABLE = "testACellDeletedNonTransactionallyIsPreservedWhenMinorCompactionOccurs";
    createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
    TTable txTable = new TTable(connection, TEST_TABLE);

    Table table = txTable.getHTable();

    // Configure the environment to create a minor compaction

    // Write first a value transactionally
    HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
    byte[] rowId = Bytes.toBytes("row1");
    Put p0 = new Put(rowId);
    p0.addColumn(fam, qual, Bytes.toBytes("testValue-0"));
    txTable.put(tx0, p0);
    tm.commit(tx0);

    // create the first hfile
    manualFlush(TEST_TABLE);

    // Write another value transactionally
    HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
    Put p1 = new Put(rowId);
    p1.addColumn(fam, qual, Bytes.toBytes("testValue-1"));
    txTable.put(tx1, p1);
    tm.commit(tx1);

    // create the second hfile
    manualFlush(TEST_TABLE);

    // Write yet another value transactionally
    HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
    Put p2 = new Put(rowId);
    p2.addColumn(fam, qual, Bytes.toBytes("testValue-2"));
    txTable.put(tx2, p2);
    tm.commit(tx2);

    // create a third hfile
    manualFlush(TEST_TABLE);

    // Then perform a non-transactional Delete
    Delete d = new Delete(rowId);
    d.addColumn(fam, qual);
    table.delete(d);

    // create the fourth hfile
    manualFlush(TEST_TABLE);

    // Trigger the minor compaction
    HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
    setCompactorLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
    admin.compact(TableName.valueOf(TEST_TABLE));
    Thread.sleep(5000);

    // Then perform a non-tx (raw) scan...
    Scan scan = new Scan();
    scan.setRaw(true);
    ResultScanner scannerResults = table.getScanner(scan);

    // ...and test the deleted cell is still there
    int count = 0;
    Result scanResult;
    List<Cell> listOfCellsScanned = new ArrayList<>();
    while ((scanResult = scannerResults.next()) != null) {
        listOfCellsScanned = scanResult.listCells(); // equivalent to rawCells()
        count++;
    }
    assertEquals(count, 1, "There should be only one result in scan results");
    assertEquals(listOfCellsScanned.size(), 3, "There should be 3 cell entries in scan results (2 puts, 1 del)");
    boolean wasDeletedCellFound = false;
    int numberOfDeletedCellsFound = 0;
    for (Cell cell : listOfCellsScanned) {
        if (CellUtil.isDelete(cell)) {
            wasDeletedCellFound = true;
            numberOfDeletedCellsFound++;
        }
    }
    assertTrue(wasDeletedCellFound, "We should have found a non-transactionally deleted cell");
    assertEquals(numberOfDeletedCellsFound, 1, "There should be only only one deleted cell");

    table.close();
}