Java Code Examples for org.apache.hadoop.hbase.wal.WALKey

The following examples show how to use org.apache.hadoop.hbase.wal.WALKey. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hbase   Source File: MetricsWAL.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void postAppend(final long size, final long time, final WALKey logkey,
    final WALEdit logEdit) throws IOException {
  source.incrementAppendCount();
  source.incrementAppendTime(time);
  source.incrementAppendSize(size);
  source.incrementWrittenBytes(size);

  if (time > 1000) {
    source.incrementSlowAppendCount();
    LOG.warn(String.format("%s took %d ms appending an edit to wal; len~=%s",
        Thread.currentThread().getName(),
        time,
        StringUtils.humanReadableInt(size)));
  }
}
 
Example 2
Source Project: hbase   Source File: WALPlayer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void map(WALKey key, WALEdit value, Context context) throws IOException {
  try {
    // skip all other tables
    TableName table = key.getTableName();
    if (tableSet.contains(table.getNameAsString())) {
      for (Cell cell : value.getCells()) {
        if (WALEdit.isMetaEditFamily(cell)) {
          continue;
        }
        byte[] outKey = multiTableSupport
            ? Bytes.add(table.getName(), Bytes.toBytes(tableSeparator), CellUtil.cloneRow(cell))
            : CellUtil.cloneRow(cell);
        context.write(new ImmutableBytesWritable(outKey), new MapReduceExtendedCell(cell));
      }
    }
  } catch (InterruptedException e) {
    LOG.error("Interrupted while emitting Cell", e);
    Thread.currentThread().interrupt();
  }
}
 
Example 3
Source Project: hbase   Source File: TestWALRecordReader.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Create a new reader from the split, and match the edits against the passed columns.
 */
private void testSplit(InputSplit split, byte[]... columns) throws Exception {
  WALRecordReader<WALKey> reader = getReader();
  reader.initialize(split, MapReduceTestUtil.createDummyMapTaskAttemptContext(conf));

  for (byte[] column : columns) {
    assertTrue(reader.nextKeyValue());
    Cell cell = reader.getCurrentValue().getCells().get(0);
    if (!Bytes.equals(column, 0, column.length, cell.getQualifierArray(),
      cell.getQualifierOffset(), cell.getQualifierLength())) {
      assertTrue(
        "expected [" + Bytes.toString(column) + "], actual [" + Bytes.toString(
          cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()) + "]",
        false);
    }
  }
  assertFalse(reader.nextKeyValue());
  reader.close();
}
 
Example 4
Source Project: hbase   Source File: ReplicationSourceWALActionListener.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Utility method used to set the correct scopes on each log key. Doesn't set a scope on keys from
 * compaction WAL edits and if the scope is local.
 * @param logKey Key that may get scoped according to its edits
 * @param logEdit Edits used to lookup the scopes
 */
@VisibleForTesting
static void scopeWALEdits(WALKey logKey, WALEdit logEdit, Configuration conf) {
  // For bulk load replication we need meta family to know the file we want to replicate.
  if (ReplicationUtils.isReplicationForBulkLoadDataEnabled(conf)) {
    return;
  }
  // For replay, or if all the cells are markers, do not need to store replication scope.
  if (logEdit.isReplay() ||
    logEdit.getCells().stream().allMatch(c -> WALEdit.isMetaEditFamily(c))) {
    ((WALKeyImpl) logKey).clearReplicationScope();
  }
}
 
Example 5
Source Project: hbase   Source File: WALCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
public void preWALWrite(final RegionInfo info, final WALKey logKey, final WALEdit logEdit)
    throws IOException {
  // Not bypassable.
  if (this.coprocEnvironments.isEmpty()) {
    return;
  }
  execOperation(new WALObserverOperation() {
    @Override
    public void call(WALObserver oserver) throws IOException {
      oserver.preWALWrite(this, info, logKey, logEdit);
    }
  });
}
 
Example 6
Source Project: hbase   Source File: WALCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
public void postWALWrite(final RegionInfo info, final WALKey logKey, final WALEdit logEdit)
    throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new WALObserverOperation() {
    @Override
    protected void call(WALObserver observer) throws IOException {
      observer.postWALWrite(this, info, logKey, logEdit);
    }
  });
}
 
Example 7
Source Project: hbase   Source File: RegionCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Supports Coprocessor 'bypass'.
 * @return true if default behavior should be bypassed, false otherwise
 * @deprecated Since hbase-2.0.0. No replacement. To be removed in hbase-3.0.0 and replaced
 * with something that doesn't expose IntefaceAudience.Private classes.
 */
@Deprecated
public boolean preWALRestore(final RegionInfo info, final WALKey logKey, final WALEdit logEdit)
    throws IOException {
  return execOperation(coprocEnvironments.isEmpty()? null:
      new RegionObserverOperationWithoutResult(true) {
    @Override
    public void call(RegionObserver observer) throws IOException {
      observer.preWALRestore(this, info, logKey, logEdit);
    }
  });
}
 
Example 8
Source Project: hbase   Source File: RegionCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @deprecated Since hbase-2.0.0. No replacement. To be removed in hbase-3.0.0 and replaced
 * with something that doesn't expose IntefaceAudience.Private classes.
 */
@Deprecated
public void postWALRestore(final RegionInfo info, final WALKey logKey, final WALEdit logEdit)
    throws IOException {
  execOperation(coprocEnvironments.isEmpty()? null:
      new RegionObserverOperationWithoutResult() {
    @Override
    public void call(RegionObserver observer) throws IOException {
      observer.postWALRestore(this, info, logKey, logEdit);
    }
  });
}
 
Example 9
Source Project: hbase   Source File: RegionCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
public void preWALAppend(WALKey key, WALEdit edit) throws IOException {
  if (this.coprocEnvironments.isEmpty()){
    return;
  }
  execOperation(new RegionObserverOperationWithoutResult() {
    @Override
    public void call(RegionObserver observer) throws IOException {
      observer.preWALAppend(this, key, edit);
    }
  });
}
 
Example 10
@Override
public void postWALWrite(ObserverContext<? extends WALCoprocessorEnvironment> ctx,
                         RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {
  // only keep primary region's edits
  if (logKey.getTableName().equals(tableName) && info.getReplicaId() == 0) {
    // Presume type is a WALKeyImpl
    entries.add(new Entry((WALKeyImpl)logKey, logEdit));
  }
}
 
Example 11
Source Project: hbase   Source File: TestRegionObserverInterface.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreWALAppend() throws Exception {
  SimpleRegionObserver sro = new SimpleRegionObserver();
  ObserverContext ctx = Mockito.mock(ObserverContext.class);
  WALKey key = new WALKeyImpl(Bytes.toBytes("region"), TEST_TABLE,
      EnvironmentEdgeManager.currentTime());
  WALEdit edit = new WALEdit();
  sro.preWALAppend(ctx, key, edit);
  Assert.assertEquals(1, key.getExtendedAttributes().size());
  Assert.assertArrayEquals(SimpleRegionObserver.WAL_EXTENDED_ATTRIBUTE_BYTES,
      key.getExtendedAttribute(Integer.toString(sro.getCtPreWALAppend())));
}
 
Example 12
Source Project: hbase   Source File: TestRegionObserverInterface.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void postAppend(long entryLen, long elapsedTimeMillis,
                       WALKey logKey, WALEdit logEdit) throws IOException {
  for (int k = 0; k < 4; k++) {
    if (!walKeysCorrect[k]) {
      walKeysCorrect[k] = Arrays.equals(SimpleRegionObserver.WAL_EXTENDED_ATTRIBUTE_BYTES,
          logKey.getExtendedAttribute(Integer.toString(k + 1)));
    }
  }
}
 
Example 13
Source Project: hbase   Source File: SimpleRegionObserver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void preWALRestore(ObserverContext<? extends RegionCoprocessorEnvironment> env,
    RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {
  String tableName = logKey.getTableName().getNameAsString();
  if (tableName.equals(TABLE_SKIPPED)) {
    // skip recovery of TABLE_SKIPPED for testing purpose
    env.bypass();
    return;
  }
  ctPreWALRestore.incrementAndGet();
}
 
Example 14
Source Project: hbase   Source File: SimpleRegionObserver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void preWALAppend(ObserverContext<RegionCoprocessorEnvironment> ctx,
                               WALKey key, WALEdit edit) throws IOException {
  ctPreWALAppend.incrementAndGet();

  key.addExtendedAttribute(Integer.toString(ctPreWALAppend.get()),
      Bytes.toBytes("foo"));
}
 
Example 15
@Override
public void postWALWrite(ObserverContext<? extends WALCoprocessorEnvironment> ctx,
                         RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {
  if (info.getTable().equals(TableName.valueOf("testCPMutationsAreWrittenToWALEdit"))) {
    savedEdit = logEdit;
  }
}
 
Example 16
Source Project: hbase   Source File: SampleRegionWALCoprocessor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void preWALWrite(ObserverContext<? extends WALCoprocessorEnvironment> env,
    RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {
  // check table name matches or not.
  if (!Bytes.equals(info.getTable().toBytes(), this.tableName)) {
    return;
  }
  preWALWriteCalled = true;
  // here we're going to remove one keyvalue from the WALEdit, and add
  // another one to it.
  List<Cell> cells = logEdit.getCells();
  Cell deletedCell = null;
  for (Cell cell : cells) {
    // assume only one kv from the WALEdit matches.
    byte[] family = CellUtil.cloneFamily(cell);
    byte[] qulifier = CellUtil.cloneQualifier(cell);

    if (Arrays.equals(family, ignoredFamily) &&
        Arrays.equals(qulifier, ignoredQualifier)) {
      LOG.debug("Found the KeyValue from WALEdit which should be ignored.");
      deletedCell = cell;
    }
    if (Arrays.equals(family, changedFamily) &&
        Arrays.equals(qulifier, changedQualifier)) {
      LOG.debug("Found the KeyValue from WALEdit which should be changed.");
      cell.getValueArray()[cell.getValueOffset()] =
          (byte) (cell.getValueArray()[cell.getValueOffset()] + 1);
    }
  }
  if (null != row) {
    cells.add(new KeyValue(row, addedFamily, addedQualifier));
  }
  if (deletedCell != null) {
    LOG.debug("About to delete a KeyValue from WALEdit.");
    cells.remove(deletedCell);
  }
}
 
Example 17
Source Project: hbase   Source File: TestHRegionServerBulkLoad.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void visitLogEntryBeforeWrite(WALKey logKey, WALEdit logEdit) {
  for (Cell cell : logEdit.getCells()) {
    KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
    for (Map.Entry entry : kv.toStringMap().entrySet()) {
      if (entry.getValue().equals(Bytes.toString(WALEdit.BULK_LOAD))) {
        found = true;
      }
    }
  }
}
 
Example 18
Source Project: hbase   Source File: TestWALLockup.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void visitLogEntryBeforeWrite(WALKey logKey, WALEdit logEdit)
    throws IOException {
  if (logKey.getTableName().getNameAsString().equalsIgnoreCase("sleep")) {
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      e.printStackTrace();
    }
  }
  if (logKey.getTableName().getNameAsString()
      .equalsIgnoreCase("DamagedWALException")) {
    throw new DamagedWALException("Failed appending");
  }
}
 
Example 19
Source Project: hbase   Source File: AbstractTestFSWAL.java    License: Apache License 2.0 5 votes vote down vote up
protected void addEdits(WAL log, RegionInfo hri, TableDescriptor htd, int times,
    MultiVersionConcurrencyControl mvcc, NavigableMap<byte[], Integer> scopes, String cf)
    throws IOException {
  final byte[] row = Bytes.toBytes(cf);
  for (int i = 0; i < times; i++) {
    long timestamp = System.currentTimeMillis();
    WALEdit cols = new WALEdit();
    cols.add(new KeyValue(row, row, row, timestamp, row));
    WALKeyImpl key = new WALKeyImpl(hri.getEncodedNameAsBytes(), htd.getTableName(),
        SequenceId.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE,
        HConstants.NO_NONCE, mvcc, scopes);
    log.appendData(hri, key, cols);
  }
  log.sync();
}
 
Example 20
Source Project: hbase   Source File: AbstractTestFSWAL.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWriteEntryCanBeNull() throws IOException {
  String testName = currentTest.getMethodName();
  AbstractFSWAL<?> wal = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(), testName,
    CONF, null, true, null, null);
  wal.close();
  TableDescriptor td = TableDescriptorBuilder.newBuilder(TableName.valueOf("table"))
    .setColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build();
  RegionInfo ri = RegionInfoBuilder.newBuilder(td.getTableName()).build();
  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
  NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  for (byte[] fam : td.getColumnFamilyNames()) {
    scopes.put(fam, 0);
  }
  long timestamp = System.currentTimeMillis();
  byte[] row = Bytes.toBytes("row");
  WALEdit cols = new WALEdit();
  cols.add(new KeyValue(row, row, row, timestamp, row));
  WALKeyImpl key =
      new WALKeyImpl(ri.getEncodedNameAsBytes(), td.getTableName(), SequenceId.NO_SEQUENCE_ID,
        timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE, HConstants.NO_NONCE, mvcc, scopes);
  try {
    wal.append(ri, key, cols, true);
    fail("Should fail since the wal has already been closed");
  } catch (IOException e) {
    // expected
    assertThat(e.getMessage(), containsString("log is closed"));
    // the WriteEntry should be null since we fail before setting it.
    assertNull(key.getWriteEntry());
  }
}
 
Example 21
Source Project: hbase   Source File: TestImportExport.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void visitLogEntryBeforeWrite(WALKey logKey, WALEdit logEdit) {
  if (logKey.getTableName().getNameAsString().equalsIgnoreCase(
      this.regionInfo.getTable().getNameAsString()) && (!logEdit.isMetaEdit())) {
    isVisited = true;
  }
}
 
Example 22
Source Project: hbase   Source File: TestWALRecordReader.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a new reader from the split, match the edits against the passed columns,
 * moving WAL to archive in between readings
 */
private void testSplitWithMovingWAL(InputSplit split, byte[] col1, byte[] col2) throws Exception {
  WALRecordReader<WALKey> reader = getReader();
  reader.initialize(split, MapReduceTestUtil.createDummyMapTaskAttemptContext(conf));

  assertTrue(reader.nextKeyValue());
  Cell cell = reader.getCurrentValue().getCells().get(0);
  if (!Bytes.equals(col1, 0, col1.length, cell.getQualifierArray(), cell.getQualifierOffset(),
    cell.getQualifierLength())) {
    assertTrue(
      "expected [" + Bytes.toString(col1) + "], actual [" + Bytes.toString(
        cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()) + "]",
      false);
  }
  // Move log file to archive directory
  // While WAL record reader is open
  WALInputFormat.WALSplit split_ = (WALInputFormat.WALSplit) split;

  Path logFile = new Path(split_.getLogFileName());
  Path archivedLog = AbstractFSWALProvider.getArchivedLogPath(logFile, conf);
  boolean result = fs.rename(logFile, archivedLog);
  assertTrue(result);
  result = fs.exists(archivedLog);
  assertTrue(result);
  assertTrue(reader.nextKeyValue());
  cell = reader.getCurrentValue().getCells().get(0);
  if (!Bytes.equals(col2, 0, col2.length, cell.getQualifierArray(), cell.getQualifierOffset(),
    cell.getQualifierLength())) {
    assertTrue(
      "expected [" + Bytes.toString(col2) + "], actual [" + Bytes.toString(
        cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()) + "]",
      false);
  }
  reader.close();
}
 
Example 23
Source Project: hbase   Source File: IntegrationTestBigLinkedList.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setup(Mapper<WALKey, WALEdit, ImmutableBytesWritable, Mutation>.Context context)
    throws IOException {
  super.setup(context);
  try {
    this.keysToFind = readKeysToSearch(context.getConfiguration());
    LOG.info("Loaded keys to find: count=" + this.keysToFind.size());
  } catch (InterruptedException e) {
    throw new InterruptedIOException(e.toString());
  }
}
 
Example 24
Source Project: hbase   Source File: IntegrationTestLoadAndVerify.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setup(Mapper<WALKey, WALEdit, ImmutableBytesWritable, Mutation>.Context context)
    throws IOException {
  super.setup(context);
  try {
    this.keysToFind = readKeysToSearch(context.getConfiguration());
    LOG.info("Loaded keys to find: count=" + this.keysToFind.size());
  } catch (InterruptedException e) {
    throw new InterruptedIOException(e.toString());
  }
}
 
Example 25
Source Project: phoenix   Source File: TestWALRecoveryCaching.java    License: Apache License 2.0 5 votes vote down vote up
@Override
    public void preWALRestore(
            org.apache.hadoop.hbase.coprocessor.ObserverContext<? extends RegionCoprocessorEnvironment> ctx,
            org.apache.hadoop.hbase.client.RegionInfo info, WALKey logKey,
            org.apache.hadoop.hbase.wal.WALEdit logEdit) throws IOException {
  try {
    LOGGER.debug("Restoring logs for index table");
    if (allowIndexTableToRecover != null) {
      allowIndexTableToRecover.await();
      LOGGER.debug("Completed index table recovery wait latch");
    }
  } catch (InterruptedException e) {
    Assert.fail("Should not be interrupted while waiting to allow the index to restore WALs.");
  }
}
 
Example 26
Source Project: hbase   Source File: ReplicationSourceWALReader.java    License: Apache License 2.0 4 votes vote down vote up
public static long getEntrySizeExcludeBulkLoad(Entry entry) {
  WALEdit edit = entry.getEdit();
  WALKey key = entry.getKey();
  return edit.heapSize() + key.estimatedSerializedSizeOf();
}
 
Example 27
Source Project: hbase   Source File: ReplicationSourceWALActionListener.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void visitLogEntryBeforeWrite(WALKey logKey, WALEdit logEdit) throws IOException {
  scopeWALEdits(logKey, logEdit, conf);
}
 
Example 28
Source Project: hbase   Source File: WALProcedurePrettyPrinter.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected int doWork() throws Exception {
  Path path = new Path(file);
  FileSystem fs = path.getFileSystem(conf);
  try (WAL.Reader reader = WALFactory.createReader(fs, path, conf)) {
    for (;;) {
      WAL.Entry entry = reader.next();
      if (entry == null) {
        return 0;
      }
      WALKey key = entry.getKey();
      WALEdit edit = entry.getEdit();
      long sequenceId = key.getSequenceId();
      long writeTime = key.getWriteTime();
      out.println(
        String.format(KEY_TMPL, sequenceId, FORMATTER.format(Instant.ofEpochMilli(writeTime))));
      for (Cell cell : edit.getCells()) {
        Map<String, Object> op = WALPrettyPrinter.toStringMap(cell);
        if (!Bytes.equals(PROC_FAMILY, 0, PROC_FAMILY.length, cell.getFamilyArray(),
          cell.getFamilyOffset(), cell.getFamilyLength())) {
          // We could have cells other than procedure edits, for example, a flush marker
          WALPrettyPrinter.printCell(out, op, false);
          continue;
        }
        long procId = Bytes.toLong(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
        out.println("pid=" + procId + ", type=" + op.get("type") + ", column=" +
          op.get("family") + ":" + op.get("qualifier"));
        if (cell.getType() == Cell.Type.Put) {
          if (cell.getValueLength() > 0) {
            // should be a normal put
            Procedure<?> proc =
              ProcedureUtil.convertToProcedure(ProcedureProtos.Procedure.parser()
                .parseFrom(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
            out.println("\t" + proc.toStringDetails());
          } else {
            // should be a 'delete' put
            out.println("\tmark deleted");
          }
        }
        out.println("cell total size sum: " + cell.heapSize());
      }
      out.println("edit heap size: " + edit.heapSize());
      out.println("position: " + reader.getPosition());
    }
  }
}
 
Example 29
Source Project: hbase   Source File: TestCoprocessorMetrics.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void postWALWrite(ObserverContext<? extends WALCoprocessorEnvironment> ctx,
                         RegionInfo info, WALKey logKey,
                         WALEdit logEdit) throws IOException {
  walEditsCount.increment();
}
 
Example 30
Source Project: hbase   Source File: SimpleRegionObserver.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void postWALRestore(ObserverContext<? extends RegionCoprocessorEnvironment> env,
                           RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {
  ctPostWALRestore.incrementAndGet();
}