org.apache.hadoop.hbase.wal.WALKey Java Examples

@Override
public void postAppend(final long size, final long time, final WALKey logkey,
    final WALEdit logEdit) throws IOException {
  source.incrementAppendCount();
  source.incrementAppendTime(time);
  source.incrementAppendSize(size);
  source.incrementWrittenBytes(size);

  if (time > 1000) {
    source.incrementSlowAppendCount();
    LOG.warn(String.format("%s took %d ms appending an edit to wal; len~=%s",
        Thread.currentThread().getName(),
        time,
        StringUtils.humanReadableInt(size)));
  }
}
 

/**
 * Create a new reader from the split, and match the edits against the passed columns.
 */
private void testSplit(InputSplit split, byte[]... columns) throws Exception {
  WALRecordReader<WALKey> reader = getReader();
  reader.initialize(split, MapReduceTestUtil.createDummyMapTaskAttemptContext(conf));

  for (byte[] column : columns) {
    assertTrue(reader.nextKeyValue());
    Cell cell = reader.getCurrentValue().getCells().get(0);
    if (!Bytes.equals(column, 0, column.length, cell.getQualifierArray(),
      cell.getQualifierOffset(), cell.getQualifierLength())) {
      assertTrue(
        "expected [" + Bytes.toString(column) + "], actual [" + Bytes.toString(
          cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()) + "]",
        false);
    }
  }
  assertFalse(reader.nextKeyValue());
  reader.close();
}
 

@Override
public void map(WALKey key, WALEdit value, Context context) throws IOException {
  try {
    // skip all other tables
    TableName table = key.getTableName();
    if (tableSet.contains(table.getNameAsString())) {
      for (Cell cell : value.getCells()) {
        if (WALEdit.isMetaEditFamily(cell)) {
          continue;
        }
        byte[] outKey = multiTableSupport
            ? Bytes.add(table.getName(), Bytes.toBytes(tableSeparator), CellUtil.cloneRow(cell))
            : CellUtil.cloneRow(cell);
        context.write(new ImmutableBytesWritable(outKey), new MapReduceExtendedCell(cell));
      }
    }
  } catch (InterruptedException e) {
    LOG.error("Interrupted while emitting Cell", e);
    Thread.currentThread().interrupt();
  }
}
 

protected void addEdits(WAL log, RegionInfo hri, TableDescriptor htd, int times,
    MultiVersionConcurrencyControl mvcc, NavigableMap<byte[], Integer> scopes, String cf)
    throws IOException {
  final byte[] row = Bytes.toBytes(cf);
  for (int i = 0; i < times; i++) {
    long timestamp = System.currentTimeMillis();
    WALEdit cols = new WALEdit();
    cols.add(new KeyValue(row, row, row, timestamp, row));
    WALKeyImpl key = new WALKeyImpl(hri.getEncodedNameAsBytes(), htd.getTableName(),
        SequenceId.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE,
        HConstants.NO_NONCE, mvcc, scopes);
    log.appendData(hri, key, cols);
  }
  log.sync();
}
 

@Test
public void testPreWALAppend() throws Exception {
  SimpleRegionObserver sro = new SimpleRegionObserver();
  ObserverContext ctx = Mockito.mock(ObserverContext.class);
  WALKey key = new WALKeyImpl(Bytes.toBytes("region"), TEST_TABLE,
      EnvironmentEdgeManager.currentTime());
  WALEdit edit = new WALEdit();
  sro.preWALAppend(ctx, key, edit);
  Assert.assertEquals(1, key.getExtendedAttributes().size());
  Assert.assertArrayEquals(SimpleRegionObserver.WAL_EXTENDED_ATTRIBUTE_BYTES,
      key.getExtendedAttribute(Integer.toString(sro.getCtPreWALAppend())));
}
 

@Override
public void postAppend(long entryLen, long elapsedTimeMillis,
                       WALKey logKey, WALEdit logEdit) throws IOException {
  for (int k = 0; k < 4; k++) {
    if (!walKeysCorrect[k]) {
      walKeysCorrect[k] = Arrays.equals(SimpleRegionObserver.WAL_EXTENDED_ATTRIBUTE_BYTES,
          logKey.getExtendedAttribute(Integer.toString(k + 1)));
    }
  }
}
 

@Override
public void preWALRestore(ObserverContext<? extends RegionCoprocessorEnvironment> env,
    RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {
  String tableName = logKey.getTableName().getNameAsString();
  if (tableName.equals(TABLE_SKIPPED)) {
    // skip recovery of TABLE_SKIPPED for testing purpose
    env.bypass();
    return;
  }
  ctPreWALRestore.incrementAndGet();
}
 

@Override
public void postWALWrite(ObserverContext<? extends WALCoprocessorEnvironment> ctx,
                         RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {
  if (info.getTable().equals(TableName.valueOf("testCPMutationsAreWrittenToWALEdit"))) {
    savedEdit = logEdit;
  }
}
 

@Override
public void preWALWrite(ObserverContext<? extends WALCoprocessorEnvironment> env,
    RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {
  // check table name matches or not.
  if (!Bytes.equals(info.getTable().toBytes(), this.tableName)) {
    return;
  }
  preWALWriteCalled = true;
  // here we're going to remove one keyvalue from the WALEdit, and add
  // another one to it.
  List<Cell> cells = logEdit.getCells();
  Cell deletedCell = null;
  for (Cell cell : cells) {
    // assume only one kv from the WALEdit matches.
    byte[] family = CellUtil.cloneFamily(cell);
    byte[] qulifier = CellUtil.cloneQualifier(cell);

    if (Arrays.equals(family, ignoredFamily) &&
        Arrays.equals(qulifier, ignoredQualifier)) {
      LOG.debug("Found the KeyValue from WALEdit which should be ignored.");
      deletedCell = cell;
    }
    if (Arrays.equals(family, changedFamily) &&
        Arrays.equals(qulifier, changedQualifier)) {
      LOG.debug("Found the KeyValue from WALEdit which should be changed.");
      cell.getValueArray()[cell.getValueOffset()] =
          (byte) (cell.getValueArray()[cell.getValueOffset()] + 1);
    }
  }
  if (null != row) {
    cells.add(new KeyValue(row, addedFamily, addedQualifier));
  }
  if (deletedCell != null) {
    LOG.debug("About to delete a KeyValue from WALEdit.");
    cells.remove(deletedCell);
  }
}
 

@Override
public void visitLogEntryBeforeWrite(WALKey logKey, WALEdit logEdit) {
  for (Cell cell : logEdit.getCells()) {
    KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
    for (Map.Entry entry : kv.toStringMap().entrySet()) {
      if (entry.getValue().equals(Bytes.toString(WALEdit.BULK_LOAD))) {
        found = true;
      }
    }
  }
}
 

@Override
public void visitLogEntryBeforeWrite(WALKey logKey, WALEdit logEdit)
    throws IOException {
  if (logKey.getTableName().getNameAsString().equalsIgnoreCase("sleep")) {
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      e.printStackTrace();
    }
  }
  if (logKey.getTableName().getNameAsString()
      .equalsIgnoreCase("DamagedWALException")) {
    throw new DamagedWALException("Failed appending");
  }
}
 

@Override
public void preWALAppend(ObserverContext<RegionCoprocessorEnvironment> ctx,
                               WALKey key, WALEdit edit) throws IOException {
  ctPreWALAppend.incrementAndGet();

  key.addExtendedAttribute(Integer.toString(ctPreWALAppend.get()),
      Bytes.toBytes("foo"));
}
 

@Test
public void testWriteEntryCanBeNull() throws IOException {
  String testName = currentTest.getMethodName();
  AbstractFSWAL<?> wal = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(), testName,
    CONF, null, true, null, null);
  wal.close();
  TableDescriptor td = TableDescriptorBuilder.newBuilder(TableName.valueOf("table"))
    .setColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build();
  RegionInfo ri = RegionInfoBuilder.newBuilder(td.getTableName()).build();
  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
  NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  for (byte[] fam : td.getColumnFamilyNames()) {
    scopes.put(fam, 0);
  }
  long timestamp = System.currentTimeMillis();
  byte[] row = Bytes.toBytes("row");
  WALEdit cols = new WALEdit();
  cols.add(new KeyValue(row, row, row, timestamp, row));
  WALKeyImpl key =
      new WALKeyImpl(ri.getEncodedNameAsBytes(), td.getTableName(), SequenceId.NO_SEQUENCE_ID,
        timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE, HConstants.NO_NONCE, mvcc, scopes);
  try {
    wal.append(ri, key, cols, true);
    fail("Should fail since the wal has already been closed");
  } catch (IOException e) {
    // expected
    assertThat(e.getMessage(), containsString("log is closed"));
    // the WriteEntry should be null since we fail before setting it.
    assertNull(key.getWriteEntry());
  }
}
 

@Override
public void visitLogEntryBeforeWrite(WALKey logKey, WALEdit logEdit) {
  if (logKey.getTableName().getNameAsString().equalsIgnoreCase(
      this.regionInfo.getTable().getNameAsString()) && (!logEdit.isMetaEdit())) {
    isVisited = true;
  }
}
 

/**
 * Create a new reader from the split, match the edits against the passed columns,
 * moving WAL to archive in between readings
 */
private void testSplitWithMovingWAL(InputSplit split, byte[] col1, byte[] col2) throws Exception {
  WALRecordReader<WALKey> reader = getReader();
  reader.initialize(split, MapReduceTestUtil.createDummyMapTaskAttemptContext(conf));

  assertTrue(reader.nextKeyValue());
  Cell cell = reader.getCurrentValue().getCells().get(0);
  if (!Bytes.equals(col1, 0, col1.length, cell.getQualifierArray(), cell.getQualifierOffset(),
    cell.getQualifierLength())) {
    assertTrue(
      "expected [" + Bytes.toString(col1) + "], actual [" + Bytes.toString(
        cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()) + "]",
      false);
  }
  // Move log file to archive directory
  // While WAL record reader is open
  WALInputFormat.WALSplit split_ = (WALInputFormat.WALSplit) split;

  Path logFile = new Path(split_.getLogFileName());
  Path archivedLog = AbstractFSWALProvider.getArchivedLogPath(logFile, conf);
  boolean result = fs.rename(logFile, archivedLog);
  assertTrue(result);
  result = fs.exists(archivedLog);
  assertTrue(result);
  assertTrue(reader.nextKeyValue());
  cell = reader.getCurrentValue().getCells().get(0);
  if (!Bytes.equals(col2, 0, col2.length, cell.getQualifierArray(), cell.getQualifierOffset(),
    cell.getQualifierLength())) {
    assertTrue(
      "expected [" + Bytes.toString(col2) + "], actual [" + Bytes.toString(
        cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()) + "]",
      false);
  }
  reader.close();
}
 

@Override
public void setup(Mapper<WALKey, WALEdit, ImmutableBytesWritable, Mutation>.Context context)
    throws IOException {
  super.setup(context);
  try {
    this.keysToFind = readKeysToSearch(context.getConfiguration());
    LOG.info("Loaded keys to find: count=" + this.keysToFind.size());
  } catch (InterruptedException e) {
    throw new InterruptedIOException(e.toString());
  }
}
 

@Override
public void setup(Mapper<WALKey, WALEdit, ImmutableBytesWritable, Mutation>.Context context)
    throws IOException {
  super.setup(context);
  try {
    this.keysToFind = readKeysToSearch(context.getConfiguration());
    LOG.info("Loaded keys to find: count=" + this.keysToFind.size());
  } catch (InterruptedException e) {
    throw new InterruptedIOException(e.toString());
  }
}
 

@Override
    public void preWALRestore(
            org.apache.hadoop.hbase.coprocessor.ObserverContext<? extends RegionCoprocessorEnvironment> ctx,
            org.apache.hadoop.hbase.client.RegionInfo info, WALKey logKey,
            org.apache.hadoop.hbase.wal.WALEdit logEdit) throws IOException {
  try {
    LOGGER.debug("Restoring logs for index table");
    if (allowIndexTableToRecover != null) {
      allowIndexTableToRecover.await();
      LOGGER.debug("Completed index table recovery wait latch");
    }
  } catch (InterruptedException e) {
    Assert.fail("Should not be interrupted while waiting to allow the index to restore WALs.");
  }
}
 

public void preWALAppend(WALKey key, WALEdit edit) throws IOException {
  if (this.coprocEnvironments.isEmpty()){
    return;
  }
  execOperation(new RegionObserverOperationWithoutResult() {
    @Override
    public void call(RegionObserver observer) throws IOException {
      observer.preWALAppend(this, key, edit);
    }
  });
}
 

/**
 * @deprecated Since hbase-2.0.0. No replacement. To be removed in hbase-3.0.0 and replaced
 * with something that doesn't expose IntefaceAudience.Private classes.
 */
@Deprecated
public void postWALRestore(final RegionInfo info, final WALKey logKey, final WALEdit logEdit)
    throws IOException {
  execOperation(coprocEnvironments.isEmpty()? null:
      new RegionObserverOperationWithoutResult() {
    @Override
    public void call(RegionObserver observer) throws IOException {
      observer.postWALRestore(this, info, logKey, logEdit);
    }
  });
}
 

@Override
public void postWALWrite(ObserverContext<? extends WALCoprocessorEnvironment> ctx,
                         RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {
  // only keep primary region's edits
  if (logKey.getTableName().equals(tableName) && info.getReplicaId() == 0) {
    // Presume type is a WALKeyImpl
    entries.add(new Entry((WALKeyImpl)logKey, logEdit));
  }
}
 

/**
 * Supports Coprocessor 'bypass'.
 * @return true if default behavior should be bypassed, false otherwise
 * @deprecated Since hbase-2.0.0. No replacement. To be removed in hbase-3.0.0 and replaced
 * with something that doesn't expose IntefaceAudience.Private classes.
 */
@Deprecated
public boolean preWALRestore(final RegionInfo info, final WALKey logKey, final WALEdit logEdit)
    throws IOException {
  return execOperation(coprocEnvironments.isEmpty()? null:
      new RegionObserverOperationWithoutResult(true) {
    @Override
    public void call(RegionObserver observer) throws IOException {
      observer.preWALRestore(this, info, logKey, logEdit);
    }
  });
}
 

public void postWALWrite(final RegionInfo info, final WALKey logKey, final WALEdit logEdit)
    throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new WALObserverOperation() {
    @Override
    protected void call(WALObserver observer) throws IOException {
      observer.postWALWrite(this, info, logKey, logEdit);
    }
  });
}
 

/**
 * Utility method used to set the correct scopes on each log key. Doesn't set a scope on keys from
 * compaction WAL edits and if the scope is local.
 * @param logKey Key that may get scoped according to its edits
 * @param logEdit Edits used to lookup the scopes
 */
@VisibleForTesting
static void scopeWALEdits(WALKey logKey, WALEdit logEdit, Configuration conf) {
  // For bulk load replication we need meta family to know the file we want to replicate.
  if (ReplicationUtils.isReplicationForBulkLoadDataEnabled(conf)) {
    return;
  }
  // For replay, or if all the cells are markers, do not need to store replication scope.
  if (logEdit.isReplay() ||
    logEdit.getCells().stream().allMatch(c -> WALEdit.isMetaEditFamily(c))) {
    ((WALKeyImpl) logKey).clearReplicationScope();
  }
}
 

public void preWALWrite(final RegionInfo info, final WALKey logKey, final WALEdit logEdit)
    throws IOException {
  // Not bypassable.
  if (this.coprocEnvironments.isEmpty()) {
    return;
  }
  execOperation(new WALObserverOperation() {
    @Override
    public void call(WALObserver oserver) throws IOException {
      oserver.preWALWrite(this, info, logKey, logEdit);
    }
  });
}
 

@Override
public void visitLogEntryBeforeWrite(WALKey logKey, WALEdit logEdit) throws IOException {
  scopeWALEdits(logKey, logEdit, conf);
}
 

@Override
public void postWALRestore(ObserverContext<? extends RegionCoprocessorEnvironment> ctx, RegionInfo info,
        WALKey logKey, WALEdit logEdit) throws IOException {
    delegate.postWALRestore(ctx, info, logKey, logEdit);
}
 

@Override
public void preWALRestore(ObserverContext<? extends RegionCoprocessorEnvironment> ctx, RegionInfo info,
        WALKey logKey, WALEdit logEdit) throws IOException {
    delegate.preWALRestore(ctx, info, logKey, logEdit);
}
 

@Override
protected int doWork() throws Exception {
  Path path = new Path(file);
  FileSystem fs = path.getFileSystem(conf);
  try (WAL.Reader reader = WALFactory.createReader(fs, path, conf)) {
    for (;;) {
      WAL.Entry entry = reader.next();
      if (entry == null) {
        return 0;
      }
      WALKey key = entry.getKey();
      WALEdit edit = entry.getEdit();
      long sequenceId = key.getSequenceId();
      long writeTime = key.getWriteTime();
      out.println(
        String.format(KEY_TMPL, sequenceId, FORMATTER.format(Instant.ofEpochMilli(writeTime))));
      for (Cell cell : edit.getCells()) {
        Map<String, Object> op = WALPrettyPrinter.toStringMap(cell);
        if (!Bytes.equals(PROC_FAMILY, 0, PROC_FAMILY.length, cell.getFamilyArray(),
          cell.getFamilyOffset(), cell.getFamilyLength())) {
          // We could have cells other than procedure edits, for example, a flush marker
          WALPrettyPrinter.printCell(out, op, false);
          continue;
        }
        long procId = Bytes.toLong(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
        out.println("pid=" + procId + ", type=" + op.get("type") + ", column=" +
          op.get("family") + ":" + op.get("qualifier"));
        if (cell.getType() == Cell.Type.Put) {
          if (cell.getValueLength() > 0) {
            // should be a normal put
            Procedure<?> proc =
              ProcedureUtil.convertToProcedure(ProcedureProtos.Procedure.parser()
                .parseFrom(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
            out.println("\t" + proc.toStringDetails());
          } else {
            // should be a 'delete' put
            out.println("\tmark deleted");
          }
        }
        out.println("cell total size sum: " + cell.heapSize());
      }
      out.println("edit heap size: " + edit.heapSize());
      out.println("position: " + reader.getPosition());
    }
  }
}
 

private WALRecordReader<WALKey> getReader() {
  return new WALKeyRecordReader();
}