Java Code Examples for org.apache.hadoop.hbase.KeyValueUtil#ensureKeyValue()

/**
 * @param put an instance of Put
 * @param timestamp  timestamp to be used as cells version
 * @param commitTimestamp  timestamp to be used as commit timestamp
 * @throws IOException if a remote or network exception occurs.
 */
static public Put markPutAsCommitted(Put put, long timestamp, long commitTimestamp) {
    final Put tsput = new Put(put.getRow(), timestamp);
    propagateAttributes(put, tsput);

    Map<byte[], List<Cell>> kvs = put.getFamilyCellMap();
    for (List<Cell> kvl : kvs.values()) {
        for (Cell c : kvl) {
            KeyValue kv = KeyValueUtil.ensureKeyValue(c);
            Bytes.putLong(kv.getValueArray(), kv.getTimestampOffset(), timestamp);
            try {
                tsput.add(kv);
            } catch (IOException e) {
                // The existing Put has this Cell, so the cloned one
                // will never throw an IOException when it's added.
                throw new RuntimeException(e);
            }
            tsput.addColumn(CellUtil.cloneFamily(kv),
                    CellUtils.addShadowCellSuffixPrefix(CellUtil.cloneQualifier(kv), 0, CellUtil.cloneQualifier(kv).length),
                    kv.getTimestamp(),
                    Bytes.toBytes(commitTimestamp));
        }
    }

    return tsput;
}
 

/**
 * Move forward on the sub-lists set previously by seek.
 * @param key seek value (should be non-null)
 * @return true if there is at least one KV to read, false otherwise
 */
@Override
public synchronized boolean reseek(Cell key) {
  /*
   * See HBASE-4195 & HBASE-3855 & HBASE-6591 for the background on this implementation. This
   * code is executed concurrently with flush and puts, without locks. Two points must be known
   * when working on this code: 1) It's not possible to use the 'kvTail' and 'snapshot'
   * variables, as they are modified during a flush. 2) The ideal implementation for performance
   * would use the sub skip list implicitly pointed by the iterators 'kvsetIt' and 'snapshotIt'.
   * Unfortunately the Java API does not offer a method to get it. So we remember the last keys
   * we iterated to and restore the reseeked set to at least that point.
   */

  KeyValue kv = KeyValueUtil.ensureKeyValue(key);
  kvsetIt = kvsetAtCreation.tailSet(getHighest(kv, kvsetItRow)).iterator();
  return seekInSubLists();
}
 

@Override
public void prePut(ObserverContext<RegionCoprocessorEnvironment> e, Put m, WALEdit edit,
    Durability durability) throws IOException {
  byte[] attribute = m.getAttribute(NON_VISIBILITY);
  byte[] cf = null;
  List<Cell> updatedCells = new ArrayList<>();
  if (attribute != null) {
    for (List<? extends Cell> edits : m.getFamilyCellMap().values()) {
      for (Cell cell : edits) {
        KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
        if (cf == null) {
          cf = CellUtil.cloneFamily(kv);
        }
        Tag tag = new ArrayBackedTag((byte) NON_VIS_TAG_TYPE, attribute);
        List<Tag> tagList = new ArrayList<>(PrivateCellUtil.getTags(cell).size() + 1);
        tagList.add(tag);
        tagList.addAll(PrivateCellUtil.getTags(cell));
        Cell newcell = PrivateCellUtil.createCell(kv, tagList);
        ((List<Cell>) updatedCells).add(newcell);
      }
    }
    m.getFamilyCellMap().remove(cf);
    // Update the family map
    m.getFamilyCellMap().put(cf, updatedCells);
  }
}
 

private Put putInternal(Transaction tx, Put put, boolean addShadowCell) throws IOException {

        throwExceptionIfOpSetsTimerange(put);

        HBaseTransaction transaction = enforceHBaseTransactionAsParam(tx);

        final long writeTimestamp = transaction.getWriteTimestamp();

        // create put with correct ts
        final Put tsput = new Put(put.getRow(), writeTimestamp);
        propagateAttributes(put, tsput);
        Map<byte[], List<Cell>> kvs = put.getFamilyCellMap();
        for (List<Cell> kvl : kvs.values()) {
            for (Cell c : kvl) {
                CellUtils.validateCell(c, writeTimestamp);
                // Reach into keyvalue to update timestamp.
                // It's not nice to reach into keyvalue internals,
                // but we want to avoid having to copy the whole thing
                KeyValue kv = KeyValueUtil.ensureKeyValue(c);
                Bytes.putLong(kv.getValueArray(), kv.getTimestampOffset(), writeTimestamp);
                tsput.add(kv);

                if (addShadowCell) {
                    tsput.addColumn(CellUtil.cloneFamily(kv),
                            CellUtils.addShadowCellSuffixPrefix(CellUtil.cloneQualifier(kv), 0, CellUtil.cloneQualifier(kv).length),
                            kv.getTimestamp(),
                            Bytes.toBytes(kv.getTimestamp()));
                } else {
                    HBaseCellId cellId = new HBaseCellId(this,
                            CellUtil.cloneRow(kv),
                            CellUtil.cloneFamily(kv),
                            CellUtil.cloneQualifier(kv),
                            kv.getTimestamp());

                    addWriteSetElement(transaction, cellId);
                }
            }
        }
        return tsput;
    }
 

/**
 * Test that we correctly rollback the state of keyvalue
 * @throws Exception
 */
@Test
@SuppressWarnings("unchecked")
public void testCorrectRollback() throws Exception {
  Put m = new Put(row);
  m.addColumn(fam, qual, ts, val);
  // setup mocks
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);

  Region region = Mockito.mock(Region.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  final byte[] stored = Bytes.toBytes("stored-value");
  final KeyValue storedKv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
  storedKv.setSequenceId(2);

  HashMap<ImmutableBytesPtr, List<Cell>> rowKeyPtrToCells =
          new  HashMap<ImmutableBytesPtr, List<Cell>>();
  rowKeyPtrToCells.put(new ImmutableBytesPtr(row), Collections.singletonList((Cell)storedKv));
  CachedLocalTable cachedLocalTable = CachedLocalTable.build(rowKeyPtrToCells);
  LocalTableState table = new LocalTableState(cachedLocalTable, m);

  // add the kvs from the mutation
  KeyValue kv = KeyValueUtil.ensureKeyValue(m.get(fam, qual).get(0));
  kv.setSequenceId(0);
  table.addPendingUpdates(kv);

  // setup the lookup
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  // check that the value is there
  Pair<CoveredDeleteScanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  Scanner s = p.getFirst();
  assertEquals("Didn't get the pending mutation's value first", kv, s.next());

  // rollback that value
  table.rollback(Arrays.asList(kv));
  p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  s = p.getFirst();
  assertEquals("Didn't correctly rollback the row - still found it!", null, s.next());
}
 

/**
 * Tests the behavior of transform() in a hierarchical filter.
 *
 * transform() only applies after a filterCell() whose return-code includes the KeyValue.
 * Lazy evaluation of AND
 */
@Test
public void testTransformMPO() throws Exception {
  // Apply the following filter:
  //     (family=fam AND qualifier=qual1 AND KeyOnlyFilter)
  //  OR (family=fam AND qualifier=qual2)
  final FilterList flist = new FilterList(Operator.MUST_PASS_ONE, Lists.<Filter>newArrayList(
      new FilterList(Operator.MUST_PASS_ALL, Lists.<Filter>newArrayList(
          new FamilyFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
          new QualifierFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("qual1"))),
          new KeyOnlyFilter())),
      new FilterList(Operator.MUST_PASS_ALL, Lists.<Filter>newArrayList(
          new FamilyFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
          new QualifierFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("qual2")))))));

  final KeyValue kvQual1 = new KeyValue(
      Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual1"), Bytes.toBytes("value"));
  final KeyValue kvQual2 = new KeyValue(
      Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual2"), Bytes.toBytes("value"));
  final KeyValue kvQual3 = new KeyValue(
      Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual3"), Bytes.toBytes("value"));

  // Value for fam:qual1 should be stripped:
  assertEquals(Filter.ReturnCode.INCLUDE, flist.filterCell(kvQual1));
  final KeyValue transformedQual1 = KeyValueUtil.ensureKeyValue(flist.transformCell(kvQual1));
  assertEquals(0, transformedQual1.getValueLength());

  // Value for fam:qual2 should not be stripped:
  assertEquals(Filter.ReturnCode.INCLUDE, flist.filterCell(kvQual2));
  final KeyValue transformedQual2 = KeyValueUtil.ensureKeyValue(flist.transformCell(kvQual2));
  assertEquals("value", Bytes.toString(transformedQual2.getValueArray(),
    transformedQual2.getValueOffset(), transformedQual2.getValueLength()));

  // Other keys should be skipped:
  assertEquals(Filter.ReturnCode.SKIP, flist.filterCell(kvQual3));
}
 

@Override
public void visitLogEntryBeforeWrite(WALKey logKey, WALEdit logEdit) {
  for (Cell cell : logEdit.getCells()) {
    KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
    for (Map.Entry entry : kv.toStringMap().entrySet()) {
      if (entry.getValue().equals(Bytes.toString(WALEdit.BULK_LOAD))) {
        found = true;
      }
    }
  }
}
 

private void updateMutationAddingTags(final Mutation m) {
  byte[] attribute = m.getAttribute("visibility");
  byte[] cf = null;
  List<Cell> updatedCells = new ArrayList<>();
  if (attribute != null) {
    for (List<? extends Cell> edits : m.getFamilyCellMap().values()) {
      for (Cell cell : edits) {
        KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
        if (cf == null) {
          cf = CellUtil.cloneFamily(kv);
        }
        Tag tag = new ArrayBackedTag((byte) 1, attribute);
        List<Tag> tagList = new ArrayList<>();
        tagList.add(tag);

        KeyValue newKV = new KeyValue(CellUtil.cloneRow(kv), 0, kv.getRowLength(),
            CellUtil.cloneFamily(kv), 0, kv.getFamilyLength(), CellUtil.cloneQualifier(kv), 0,
            kv.getQualifierLength(), kv.getTimestamp(),
            KeyValue.Type.codeToType(kv.getTypeByte()), CellUtil.cloneValue(kv), 0,
            kv.getValueLength(), tagList);
        ((List<Cell>) updatedCells).add(newKV);
      }
    }
    m.getFamilyCellMap().remove(cf);
    // Update the family map
    m.getFamilyCellMap().put(cf, updatedCells);
  }
}
 

@Override
public void prePut(final ObserverContext<RegionCoprocessorEnvironment> e, final Put put,
    final WALEdit edit, final Durability durability) throws IOException {
  byte[] attribute = put.getAttribute("visibility");
  byte[] cf = null;
  List<Cell> updatedCells = new ArrayList<>();
  if (attribute != null) {
    for (List<? extends Cell> edits : put.getFamilyCellMap().values()) {
      for (Cell cell : edits) {
        KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
        if (cf == null) {
          cf = CellUtil.cloneFamily(kv);
        }
        Tag tag = new ArrayBackedTag(TAG_TYPE, attribute);
        List<Tag> tagList = new ArrayList<>(1);
        tagList.add(tag);

        KeyValue newKV = new KeyValue(CellUtil.cloneRow(kv), 0, kv.getRowLength(),
            CellUtil.cloneFamily(kv), 0, kv.getFamilyLength(), CellUtil.cloneQualifier(kv), 0,
            kv.getQualifierLength(), kv.getTimestamp(),
            KeyValue.Type.codeToType(kv.getTypeByte()), CellUtil.cloneValue(kv), 0,
            kv.getValueLength(), tagList);
        ((List<Cell>) updatedCells).add(newKV);
      }
    }
    put.getFamilyCellMap().remove(cf);
    // Update the family map
    put.getFamilyCellMap().put(cf, updatedCells);
  }
}
 

@SuppressWarnings("deprecation")
@Override
protected void writeToBuffer(MappedByteBuffer buffer, Tuple e) {
    KeyValue kv = KeyValueUtil.ensureKeyValue(e.getValue(0));
    buffer.putInt(kv.getLength() + Bytes.SIZEOF_INT);
    buffer.putInt(kv.getLength());
    buffer.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
}
 

/**
 * Test that we correctly rollback the state of keyvalue
 * @throws Exception
 */
@Test
@SuppressWarnings("unchecked")
public void testCorrectRollback() throws Exception {
  Put m = new Put(row);
  m.add(fam, qual, ts, val);
  // setup mocks
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);

  HRegion region = Mockito.mock(HRegion.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  RegionScanner scanner = Mockito.mock(RegionScanner.class);
  Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner);
  final byte[] stored = Bytes.toBytes("stored-value");
  final KeyValue storedKv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
  storedKv.setSequenceId(2);
  Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer<Boolean>() {
    @Override
    public Boolean answer(InvocationOnMock invocation) throws Throwable {
      List<KeyValue> list = (List<KeyValue>) invocation.getArguments()[0];

      list.add(storedKv);
      return false;
    }
  });
  LocalHBaseState state = new LocalTable(env);
  LocalTableState table = new LocalTableState(env, state, m);
  // add the kvs from the mutation
  KeyValue kv = KeyValueUtil.ensureKeyValue(m.get(fam, qual).get(0));
  kv.setSequenceId(0);
  table.addPendingUpdates(kv);

  // setup the lookup
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  // check that the value is there
  Pair<Scanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col));
  Scanner s = p.getFirst();
  assertEquals("Didn't get the pending mutation's value first", kv, s.next());

  // rollback that value
  table.rollback(Arrays.asList(kv));
  p = table.getIndexedColumnsTableState(Arrays.asList(col));
  s = p.getFirst();
  assertEquals("Didn't correctly rollback the row - still found it!", null, s.next());
  Mockito.verify(env, Mockito.times(1)).getRegion();
  Mockito.verify(region, Mockito.times(1)).getScanner(Mockito.any(Scan.class));
}
 

/**
 * Update the current statistics based on the latest batch of key-values from the underlying scanner
 * 
 * @param results
 *            next batch of {@link KeyValue}s
 */
public void collectStatistics(final List<Cell> results) {
    Map<ImmutableBytesPtr, Boolean> famMap = Maps.newHashMap();
    List<GuidePostsInfo> rowTracker = null;
    if(cachedGps == null) {
        rowTracker = 
                new ArrayList<GuidePostsInfo>();
    }
    for (Cell cell : results) {
        KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
        maxTimeStamp = Math.max(maxTimeStamp, kv.getTimestamp());
        Pair<Long, GuidePostsInfo> gps;
        if (cachedGps == null) {
            ImmutableBytesPtr cfKey = new ImmutableBytesPtr(kv.getFamilyArray(), kv.getFamilyOffset(),
                    kv.getFamilyLength());
            gps = guidePostsMap.get(cfKey);
            if (gps == null) {
                gps = new Pair<Long, GuidePostsInfo>(0l, new GuidePostsInfo(0,
                        Collections.<byte[]> emptyList(), 0l));
                guidePostsMap.put(cfKey, gps);
            }
            if (famMap.get(cfKey) == null) {
                famMap.put(cfKey, true);
                rowTracker.add(gps.getSecond());
            }
        } else {
            gps = cachedGps;
        }
        int kvLength = kv.getLength();
        long byteCount = gps.getFirst() + kvLength;
        gps.setFirst(byteCount);
        if (byteCount >= guidepostDepth) {
            byte[] row = ByteUtil.copyKeyBytesIfNecessary(new ImmutableBytesWritable(kv.getRowArray(), kv
                    .getRowOffset(), kv.getRowLength()));
            if (gps.getSecond().addGuidePost(row, byteCount)) {
                gps.setFirst(0l);
            }
        }
    }
    if(cachedGps == null) {
        for (GuidePostsInfo s : rowTracker) {
            s.incrementRowCount();
        }
    } else {
        cachedGps.getSecond().incrementRowCount();
    }
}
 

/**
 * Check statistics for given HFile for different data block encoders.
 * @param scanner Of file which will be compressed.
 * @param kvLimit Maximal count of KeyValue which will be processed.
 * @throws IOException thrown if scanner is invalid
 */
public void checkStatistics(final KeyValueScanner scanner, final int kvLimit)
    throws IOException {
  scanner.seek(KeyValue.LOWESTKEY);

  KeyValue currentKV;

  byte[] previousKey = null;
  byte[] currentKey;

  DataBlockEncoding[] encodings = DataBlockEncoding.values();

  ByteArrayOutputStream uncompressedOutputStream =
      new ByteArrayOutputStream();

  int j = 0;
  while ((currentKV = KeyValueUtil.ensureKeyValue(scanner.next())) != null && j < kvLimit) {
    // Iterates through key/value pairs
    j++;
    currentKey = currentKV.getKey();
    if (previousKey != null) {
      for (int i = 0; i < previousKey.length && i < currentKey.length &&
          previousKey[i] == currentKey[i]; ++i) {
        totalKeyRedundancyLength++;
      }
    }

    // Add tagsLen zero to cells don't include tags. Since the process of
    // scanner converts byte array to KV would abandon tagsLen part if tagsLen
    // is zero. But we still needs the tagsLen part to check if current cell
    // include tags. If USE_TAG is true, HFile contains cells with tags,
    // if the cell tagsLen equals 0, it means other cells may have tags.
    if (USE_TAG && currentKV.getTagsLength() == 0) {
      uncompressedOutputStream.write(currentKV.getBuffer(),
          currentKV.getOffset(), currentKV.getLength());
      // write tagsLen = 0.
      uncompressedOutputStream.write(Bytes.toBytes((short) 0));
    } else {
      uncompressedOutputStream.write(currentKV.getBuffer(),
          currentKV.getOffset(), currentKV.getLength());
    }

    if(includesMemstoreTS) {
      WritableUtils.writeVLong(
          new DataOutputStream(uncompressedOutputStream), currentKV.getSequenceId());
    }

    previousKey = currentKey;

    int kLen = currentKV.getKeyLength();
    int vLen = currentKV.getValueLength();
    int cfLen = currentKV.getFamilyLength(currentKV.getFamilyOffset());
    int restLen = currentKV.getLength() - kLen - vLen;

    totalKeyLength += kLen;
    totalValueLength += vLen;
    totalPrefixLength += restLen;
    totalCFLength += cfLen;
  }

  rawKVs = uncompressedOutputStream.toByteArray();
  for (DataBlockEncoding encoding : encodings) {
    if (encoding == DataBlockEncoding.NONE) {
      continue;
    }
    DataBlockEncoder d = encoding.getEncoder();
    HFileContext meta = new HFileContextBuilder()
        .withDataBlockEncoding(encoding)
        .withCompression(Compression.Algorithm.NONE)
        .withIncludesMvcc(includesMemstoreTS)
        .withIncludesTags(USE_TAG).build();
    codecs.add(new EncodedDataBlock(d, encoding, rawKVs, meta ));
  }
}
 

/**
 * Verify if all data block encoders are working properly.
 *
 * @param scanner Of file which was compressed.
 * @param kvLimit Maximal count of KeyValue which will be processed.
 * @return true if all data block encoders compressed/decompressed correctly.
 * @throws IOException thrown if scanner is invalid
 */
public boolean verifyCodecs(final KeyValueScanner scanner, final int kvLimit)
    throws IOException {
  KeyValue currentKv;

  scanner.seek(KeyValue.LOWESTKEY);
  List<Iterator<Cell>> codecIterators = new ArrayList<>();
  for(EncodedDataBlock codec : codecs) {
    codecIterators.add(codec.getIterator(HFileBlock.headerSize(useHBaseChecksum)));
  }

  int j = 0;
  while ((currentKv = KeyValueUtil.ensureKeyValue(scanner.next())) != null && j < kvLimit) {
    // Iterates through key/value pairs
    ++j;
    for (Iterator<Cell> it : codecIterators) {
      Cell c = it.next();
      KeyValue codecKv = KeyValueUtil.ensureKeyValue(c);
      if (codecKv == null || 0 != Bytes.compareTo(
          codecKv.getBuffer(), codecKv.getOffset(), codecKv.getLength(),
          currentKv.getBuffer(), currentKv.getOffset(),
          currentKv.getLength())) {
        if (codecKv == null) {
          LOG.error("There is a bug in codec " + it +
              " it returned null KeyValue,");
        } else {
          int prefix = 0;
          int limitLength = 2 * Bytes.SIZEOF_INT +
              Math.min(codecKv.getLength(), currentKv.getLength());
          while (prefix < limitLength &&
              codecKv.getBuffer()[prefix + codecKv.getOffset()] ==
              currentKv.getBuffer()[prefix + currentKv.getOffset()]) {
            prefix++;
          }

          LOG.error("There is bug in codec " + it.toString() +
              "\n on element " + j +
              "\n codecKv.getKeyLength() " + codecKv.getKeyLength() +
              "\n codecKv.getValueLength() " + codecKv.getValueLength() +
              "\n codecKv.getLength() " + codecKv.getLength() +
              "\n currentKv.getKeyLength() " + currentKv.getKeyLength() +
              "\n currentKv.getValueLength() " + currentKv.getValueLength() +
              "\n codecKv.getLength() " + currentKv.getLength() +
              "\n currentKV rowLength " + currentKv.getRowLength() +
              " familyName " + currentKv.getFamilyLength() +
              " qualifier " + currentKv.getQualifierLength() +
              "\n prefix " + prefix +
              "\n codecKv   '" + Bytes.toStringBinary(codecKv.getBuffer(),
                  codecKv.getOffset(), prefix) + "' diff '" +
                  Bytes.toStringBinary(codecKv.getBuffer(),
                      codecKv.getOffset() + prefix, codecKv.getLength() -
                      prefix) + "'" +
              "\n currentKv '" + Bytes.toStringBinary(
                 currentKv.getBuffer(),
                 currentKv.getOffset(), prefix) + "' diff '" +
                 Bytes.toStringBinary(currentKv.getBuffer(),
                     currentKv.getOffset() + prefix, currentKv.getLength() -
                     prefix) + "'"
              );
        }
        return false;
      }
    }
  }

  LOG.info("Verification was successful!");

  return true;
}
 

@Override
protected int sizeOf(Tuple e) {
    KeyValue kv = KeyValueUtil.ensureKeyValue(e.getValue(0));
    return Bytes.SIZEOF_INT * 2 + kv.getLength();
}
 

@Override
protected void reduce(K row, Iterable<Put> vals, Context context)
    throws IOException, InterruptedException {
  // Using HeapSize to create an upper bound on the memory size of
  // the puts and flush some portion of the content while looping. This
  // flush could result in multiple Puts for a single rowkey. That is
  // acceptable because Combiner is run as an optimization and it's not
  // critical that all Puts are grouped perfectly.
  long threshold = context.getConfiguration().getLong(
      "putcombiner.row.threshold", 1L * (1<<30));
  int cnt = 0;
  long curSize = 0;
  Put put = null;
  Map<byte[], List<Cell>> familyMap = null;
  for (Put p : vals) {
    cnt++;
    if (put == null) {
      put = p;
      familyMap = put.getFamilyCellMap();
    } else {
      for (Entry<byte[], List<Cell>> entry : p.getFamilyCellMap()
          .entrySet()) {
        List<Cell> cells = familyMap.get(entry.getKey());
        List<Cell> kvs = (cells != null) ? (List<Cell>) cells : null;
        for (Cell cell : entry.getValue()) {
          KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
          curSize += kv.heapSize();
          if (kvs != null) {
            kvs.add(kv);
          }
        }
        if (cells == null) {
          familyMap.put(entry.getKey(), entry.getValue());
        }
      }
      if (cnt % 10 == 0) context.setStatus("Combine " + cnt);
      if (curSize > threshold) {
        if (LOG.isDebugEnabled()) {
          LOG.debug(String.format("Combined %d Put(s) into %d.", cnt, 1));
        }
        context.write(row, put);
        put = null;
        curSize = 0;
        cnt = 0;
      }
    }
  }
  if (put != null) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(String.format("Combined %d Put(s) into %d.", cnt, 1));
    }
    context.write(row, put);
  }
}
 

@Override
public ReturnCode filterKeyValue(Cell next) {
  // we marked ourselves done, but the END_ROW_KEY didn't manage to seek to the very last key
  if (this.done) {
    return ReturnCode.SKIP;
  }

  KeyValue nextKV = KeyValueUtil.ensureKeyValue(next);
  switch (KeyValue.Type.codeToType(next.getTypeByte())) {
  /*
   * DeleteFamily will always sort first because those KVs (we assume) don't have qualifiers (or
   * rather are null). Therefore, we have to keep a hold of all the delete families until we get
   * to a Put entry that is covered by that delete (in which case, we are done with the family).
   */
  case DeleteFamily:
    // track the family to delete. If we are updating the delete, that means we have passed all
    // kvs in the last column, so we can safely ignore the last deleteFamily, and just use this
    // one. In fact, it means that all the previous deletes can be ignored because the family must
    // not match anymore.
    this.coveringDelete.reset();
    this.coveringDelete.deleteFamily = nextKV;
    return ReturnCode.SKIP;
  case DeleteColumn:
    // similar to deleteFamily, all the newer deletes/puts would have been seen at this point, so
    // we can safely replace the more recent delete column with the more recent one
    this.coveringDelete.pointDelete = null;
    this.coveringDelete.deleteColumn = nextKV;
    return ReturnCode.SKIP;
  case Delete:
    // we are just deleting the single column value at this point.
    // therefore we just skip this entry and go onto the next one. The only caveat is that
    // we should still cover the next entry if this delete applies to the next entry, so we
    // have to keep around a reference to the KV to compare against the next valid entry
    this.coveringDelete.pointDelete = nextKV;
    return ReturnCode.SKIP;
  default:
    // no covering deletes
    if (coveringDelete.empty()) {
      return ReturnCode.INCLUDE;
    }

    if (coveringDelete.matchesFamily(nextKV)) {
      this.currentHint = familyHint;
      return ReturnCode.SEEK_NEXT_USING_HINT;
    }

    if (coveringDelete.matchesColumn(nextKV)) {
      // hint to the next column
      this.currentHint = columnHint;
      return ReturnCode.SEEK_NEXT_USING_HINT;
    }

    if (coveringDelete.matchesPoint(nextKV)) {
      return ReturnCode.SKIP;
    }

  }

  // none of the deletes matches, we are done
  return ReturnCode.INCLUDE;
}
 

@Override
public AdminProtos.ReplicateWALEntryResponse replicateWALEntry(final RpcController controller,
        final AdminProtos.ReplicateWALEntryRequest request) throws ServiceException {
    try {

        // TODO Recording of last processed timestamp won't work if two batches of log entries are sent out of order
        long lastProcessedTimestamp = -1;

        SepEventExecutor eventExecutor = new SepEventExecutor(listener, executors, 100, sepMetrics);

        List<AdminProtos.WALEntry> entries = request.getEntryList();
        CellScanner cells = ((PayloadCarryingRpcController)controller).cellScanner();

        for (final AdminProtos.WALEntry entry : entries) {
            TableName tableName = (entry.getKey().getWriteTime() < subscriptionTimestamp) ? null :
                    TableName.valueOf(entry.getKey().getTableName().toByteArray());
            Multimap<ByteBuffer, Cell> keyValuesPerRowKey = ArrayListMultimap.create();
            final Map<ByteBuffer, byte[]> payloadPerRowKey = Maps.newHashMap();
            int count = entry.getAssociatedCellCount();
            for (int i = 0; i < count; i++) {
                if (!cells.advance()) {
                    throw new ArrayIndexOutOfBoundsException("Expected=" + count + ", index=" + i);
                }

                // this signals to us that we simply need to skip over count of cells
                if (tableName == null) {
                    continue;
                }

                Cell cell = cells.current();
                ByteBuffer rowKey = ByteBuffer.wrap(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
                byte[] payload;
                KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
                if (payloadExtractor != null && (payload = payloadExtractor.extractPayload(tableName.toBytes(), kv)) != null) {
                    if (payloadPerRowKey.containsKey(rowKey)) {
                        log.error("Multiple payloads encountered for row " + Bytes.toStringBinary(rowKey)
                                + ", choosing " + Bytes.toStringBinary(payloadPerRowKey.get(rowKey)));
                    } else {
                        payloadPerRowKey.put(rowKey, payload);
                    }
                }
                keyValuesPerRowKey.put(rowKey, kv);
            }

            for (final ByteBuffer rowKeyBuffer : keyValuesPerRowKey.keySet()) {
                final List<Cell> keyValues = (List<Cell>) keyValuesPerRowKey.get(rowKeyBuffer);

                final SepEvent sepEvent = new SepEvent(tableName.toBytes(), CellUtil.cloneRow(keyValues.get(0)), keyValues,
                        payloadPerRowKey.get(rowKeyBuffer));
                eventExecutor.scheduleSepEvent(sepEvent);
                lastProcessedTimestamp = Math.max(lastProcessedTimestamp, entry.getKey().getWriteTime());
            }

        }
        List<Future<?>> futures = eventExecutor.flush();
        waitOnSepEventCompletion(futures);

        if (lastProcessedTimestamp > 0) {
            sepMetrics.reportSepTimestamp(lastProcessedTimestamp);
        }
        return AdminProtos.ReplicateWALEntryResponse.newBuilder().build();
    } catch (IOException ie) {
        throw new ServiceException(ie);
    }
}