org.apache.hadoop.hbase.wal.WALEdit Java Examples

The following examples show how to use org.apache.hadoop.hbase.wal.WALEdit. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestBulkLoad.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void shouldBulkLoadManyFamilyHLogEvenWhenTableNameNamespaceSpecified() throws IOException {
  when(log.appendMarker(any(),
          any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)))).thenAnswer(new Answer() {
            @Override
            public Object answer(InvocationOnMock invocation) {
              WALKeyImpl walKey = invocation.getArgument(1);
              MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
              if (mvcc != null) {
                MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
                walKey.setWriteEntry(we);
              }
              return 01L;
            }
  });
  TableName tableName = TableName.valueOf("test", "test");
  testRegionWithFamiliesAndSpecifiedTableName(tableName, family1, family2)
      .bulkLoadHFiles(withFamilyPathsFor(family1, family2), false, null);
  verify(log).sync(anyLong());
}
 
Example #2
Source File: RangerAuthorizationCoprocessor.java    From ranger with Apache License 2.0 6 votes vote down vote up
@Override
public void postDelete(ObserverContext<RegionCoprocessorEnvironment> c,	Delete delete, WALEdit edit, Durability durability)	throws IOException {
	if(LOG.isDebugEnabled()) {
		LOG.debug("==> RangerAuthorizationCoprocessor.postDelete()");
	}

	try {
		activatePluginClassLoader();
		implRegionObserver.postDelete(c, delete, edit, durability);
	} finally {
		deactivatePluginClassLoader();
	}

	if(LOG.isDebugEnabled()) {
		LOG.debug("<== RangerAuthorizationCoprocessor.postDelete()");
	}
}
 
Example #3
Source File: TestBulkLoad.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
protected boolean matchesSafely(WALEdit item) {
  assertTrue(Arrays.equals(CellUtil.cloneQualifier(item.getCells().get(0)), typeBytes));
  BulkLoadDescriptor desc;
  try {
    desc = WALEdit.getBulkLoadDescriptor(item.getCells().get(0));
  } catch (IOException e) {
    return false;
  }
  assertNotNull(desc);

  if (tableName != null) {
    assertTrue(Bytes.equals(ProtobufUtil.toTableName(desc.getTableName()).getName(),
      tableName));
  }

  if(storeFileNames != null) {
    int index=0;
    StoreDescriptor store = desc.getStores(0);
    assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), familyName));
    assertTrue(Bytes.equals(Bytes.toBytes(store.getStoreHomeDir()), familyName));
    assertEquals(storeFileNames.size(), store.getStoreFileCount());
  }

  return true;
}
 
Example #4
Source File: RangerAuthorizationCoprocessor.java    From ranger with Apache License 2.0 6 votes vote down vote up
@Override
public void preDelete(ObserverContext<RegionCoprocessorEnvironment> c, Delete delete, WALEdit edit, Durability durability) throws IOException {
	if(LOG.isDebugEnabled()) {
		LOG.debug("==> RangerAuthorizationCoprocessor.preDelete()");
	}

	try {
		activatePluginClassLoader();
		implRegionObserver.preDelete(c, delete, edit, durability);
	} finally {
		deactivatePluginClassLoader();
	}

	if(LOG.isDebugEnabled()) {
		LOG.debug("<== RangerAuthorizationCoprocessor.preDelete()");
	}
}
 
Example #5
Source File: ClusterMarkingEntryFilter.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public Entry filter(Entry entry) {
  // don't replicate if the log entries have already been consumed by the cluster
  if (replicationEndpoint.canReplicateToSameCluster()
      || !entry.getKey().getClusterIds().contains(peerClusterId)) {
    WALEdit edit = entry.getEdit();
    WALKeyImpl logKey = (WALKeyImpl)entry.getKey();

    if (edit != null && !edit.isEmpty()) {
      // Mark that the current cluster has the change
      logKey.addClusterId(clusterId);
      return entry;
    }
  }
  return null;
}
 
Example #6
Source File: ReplicationSourceShipper.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void cleanUpHFileRefs(WALEdit edit) throws IOException {
  String peerId = source.getPeerId();
  if (peerId.contains("-")) {
    // peerClusterZnode will be in the form peerId + "-" + rsZNode.
    // A peerId will not have "-" in its name, see HBASE-11394
    peerId = peerId.split("-")[0];
  }
  List<Cell> cells = edit.getCells();
  int totalCells = cells.size();
  for (int i = 0; i < totalCells; i++) {
    Cell cell = cells.get(i);
    if (CellUtil.matchingQualifier(cell, WALEdit.BULK_LOAD)) {
      BulkLoadDescriptor bld = WALEdit.getBulkLoadDescriptor(cell);
      List<StoreDescriptor> stores = bld.getStoresList();
      int totalStores = stores.size();
      for (int j = 0; j < totalStores; j++) {
        List<String> storeFileList = stores.get(j).getStoreFileList();
        source.getSourceManager().cleanUpHFileRefs(peerId, storeFileList);
        source.getSourceMetrics().decrSizeOfHFileRefsQueue(storeFileList.size());
      }
    }
  }
}
 
Example #7
Source File: TestHBaseInterClusterReplicationEndpointFilterEdits.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testFilterNotExistColumnFamilyEdits() {
  List<List<Entry>> entryList = new ArrayList<>();
  // should be filtered
  Cell c1 = new KeyValue(ROW, NON_EXISTING_FAMILY, QUALIFIER, System.currentTimeMillis(),
      Type.Put, VALUE);
  Entry e1 = new Entry(new WALKeyImpl(new byte[32], TABLE1, System.currentTimeMillis()),
      new WALEdit().add(c1));
  entryList.add(Lists.newArrayList(e1));
  // should be kept
  Cell c2 = new KeyValue(ROW, FAMILY, QUALIFIER, System.currentTimeMillis(), Type.Put, VALUE);
  Entry e2 = new Entry(new WALKeyImpl(new byte[32], TABLE1, System.currentTimeMillis()),
      new WALEdit().add(c2));
  entryList.add(Lists.newArrayList(e2, e1));
  List<List<Entry>> filtered = endpoint.filterNotExistColumnFamilyEdits(entryList);
  assertEquals(1, filtered.size());
  assertEquals(1, filtered.get(0).get(0).getEdit().getCells().size());
  Cell cell = filtered.get(0).get(0).getEdit().getCells().get(0);
  assertTrue(CellUtil.matchingFamily(cell, FAMILY));
}
 
Example #8
Source File: TestHBaseInterClusterReplicationEndpointFilterEdits.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testFilterNotExistTableEdits() {
  List<List<Entry>> entryList = new ArrayList<>();
  // should be filtered
  Cell c1 = new KeyValue(ROW, FAMILY, QUALIFIER, System.currentTimeMillis(), Type.Put, VALUE);
  Entry e1 = new Entry(new WALKeyImpl(new byte[32], TABLE2, System.currentTimeMillis()),
      new WALEdit().add(c1));
  entryList.add(Lists.newArrayList(e1));
  // should be kept
  Cell c2 = new KeyValue(ROW, FAMILY, QUALIFIER, System.currentTimeMillis(), Type.Put, VALUE);
  Entry e2 = new Entry(new WALKeyImpl(new byte[32], TABLE1, System.currentTimeMillis()),
      new WALEdit().add(c2));
  entryList.add(Lists.newArrayList(e2));
  List<List<Entry>> filtered = endpoint.filterNotExistTableEdits(entryList);
  assertEquals(1, filtered.size());
  Entry entry = filtered.get(0).get(0);
  assertEquals(1, entry.getEdit().getCells().size());
  assertEquals(TABLE1, entry.getKey().getTableName());
}
 
Example #9
Source File: FSWALEntry.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * @param inMemstore If true, then this is a data edit, one that came from client. If false, it
 *   is a meta edit made by the hbase system itself and is for the WAL only.
 */
FSWALEntry(final long txid, final WALKeyImpl key, final WALEdit edit, final RegionInfo regionInfo,
  final boolean inMemstore, ServerCall<?> rpcCall) {
  super(key, edit);
  this.inMemstore = inMemstore;
  this.closeRegion = !inMemstore && edit.isRegionCloseMarker();
  this.regionInfo = regionInfo;
  this.txid = txid;
  if (inMemstore) {
    // construct familyNames here to reduce the work of log sinker.
    Set<byte[]> families = edit.getFamilies();
    this.familyNames = families != null ? families : collectFamilies(edit.getCells());
  } else {
    this.familyNames = Collections.emptySet();
  }
  this.rpcCall = rpcCall;
  if (rpcCall != null) {
    rpcCall.retainByWAL();
  }
}
 
Example #10
Source File: AbstractFSWAL.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected final long stampSequenceIdAndPublishToRingBuffer(RegionInfo hri, WALKeyImpl key,
  WALEdit edits, boolean inMemstore, RingBuffer<RingBufferTruck> ringBuffer)
  throws IOException {
  if (this.closed) {
    throw new IOException(
      "Cannot append; log is closed, regionName = " + hri.getRegionNameAsString());
  }
  MutableLong txidHolder = new MutableLong();
  MultiVersionConcurrencyControl.WriteEntry we = key.getMvcc().begin(() -> {
    txidHolder.setValue(ringBuffer.next());
  });
  long txid = txidHolder.longValue();
  ServerCall<?> rpcCall = RpcServer.getCurrentCall().filter(c -> c instanceof ServerCall)
    .filter(c -> c.getCellScanner() != null).map(c -> (ServerCall) c).orElse(null);
  try (TraceScope scope = TraceUtil.createTrace(implClassName + ".append")) {
    FSWALEntry entry = new FSWALEntry(txid, key, edits, hri, inMemstore, rpcCall);
    entry.stampRegionSequenceId(we);
    ringBuffer.get(txid).load(entry);
  } finally {
    ringBuffer.publish(txid);
  }
  return txid;
}
 
Example #11
Source File: TestBulkLoad.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void shouldBulkLoadManyFamilyHLog() throws IOException {
  when(log.appendMarker(any(),
          any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD)))).thenAnswer(new Answer() {
            @Override
            public Object answer(InvocationOnMock invocation) {
              WALKeyImpl walKey = invocation.getArgument(1);
              MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
              if (mvcc != null) {
                MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
                walKey.setWriteEntry(we);
              }
              return 01L;
            }
          });
  testRegionWithFamilies(family1, family2).bulkLoadHFiles(withFamilyPathsFor(family1, family2),
          false, null);
  verify(log).sync(anyLong());
}
 
Example #12
Source File: WALPlayer.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public void map(WALKey key, WALEdit value, Context context) throws IOException {
  try {
    // skip all other tables
    TableName table = key.getTableName();
    if (tableSet.contains(table.getNameAsString())) {
      for (Cell cell : value.getCells()) {
        if (WALEdit.isMetaEditFamily(cell)) {
          continue;
        }
        byte[] outKey = multiTableSupport
            ? Bytes.add(table.getName(), Bytes.toBytes(tableSeparator), CellUtil.cloneRow(cell))
            : CellUtil.cloneRow(cell);
        context.write(new ImmutableBytesWritable(outKey), new MapReduceExtendedCell(cell));
      }
    }
  } catch (InterruptedException e) {
    LOG.error("Interrupted while emitting Cell", e);
    Thread.currentThread().interrupt();
  }
}
 
Example #13
Source File: MetricsWAL.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public void postAppend(final long size, final long time, final WALKey logkey,
    final WALEdit logEdit) throws IOException {
  source.incrementAppendCount();
  source.incrementAppendTime(time);
  source.incrementAppendSize(size);
  source.incrementWrittenBytes(size);

  if (time > 1000) {
    source.incrementSlowAppendCount();
    LOG.warn(String.format("%s took %d ms appending an edit to wal; len~=%s",
        Thread.currentThread().getName(),
        time,
        StringUtils.humanReadableInt(size)));
  }
}
 
Example #14
Source File: SIObserver.java    From spliceengine with GNU Affero General Public License v3.0 6 votes vote down vote up
@Override
public void preDelete(ObserverContext<RegionCoprocessorEnvironment> c, Delete delete, WALEdit edit, Durability durability) throws IOException {
    checkAccess();

    try {
        if(tableEnvMatch){
            if(delete.getAttribute(SIConstants.SUPPRESS_INDEXING_ATTRIBUTE_NAME)==null){
                TableName tableName=c.getEnvironment().getRegion().getTableDescriptor().getTableName();
                String message="Direct deletes are not supported under snapshot isolation. "+
                        "Instead a Put is expected that will set a record level tombstone. tableName="+tableName;
                throw new RuntimeException(message);
            }
        }
    } catch (Throwable t) {
        throw CoprocessorUtils.getIOException(t);
    }
}
 
Example #15
Source File: TestReplicationEndpoint.java    From hbase with Apache License 2.0 5 votes vote down vote up
private Entry createEntry(String tableName, TreeMap<byte[], Integer> scopes, byte[]... kvs) {
  WALKeyImpl key1 = new WALKeyImpl(new byte[0], TableName.valueOf(tableName),
      System.currentTimeMillis() - 1L,
      scopes);
  WALEdit edit1 = new WALEdit();

  for (byte[] kv : kvs) {
    edit1.add(new KeyValue(kv, kv, kv));
  }
  return new Entry(key1, edit1);
}
 
Example #16
Source File: IntegrationTestBigLinkedList.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(Mapper<WALKey, WALEdit, ImmutableBytesWritable, Mutation>.Context context)
    throws IOException {
  super.setup(context);
  try {
    this.keysToFind = readKeysToSearch(context.getConfiguration());
    LOG.info("Loaded keys to find: count=" + this.keysToFind.size());
  } catch (InterruptedException e) {
    throw new InterruptedIOException(e.toString());
  }
}
 
Example #17
Source File: TransactionAwareHTableTest.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@Override

    public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c, final Put put,
        final WALEdit edit, final Durability durability) throws IOException {
      if (put.getAttribute(TEST_ATTRIBUTE) == null) {
        throw new DoNotRetryIOException("Put should preserve attributes");
      }
      if (put.getDurability() != Durability.USE_DEFAULT) {
        throw new DoNotRetryIOException("Durability is not propagated correctly");
      }
    }
 
Example #18
Source File: TestReplicationWALEntryFilters.java    From hbase with Apache License 2.0 5 votes vote down vote up
private Entry createEntry(TreeMap<byte[], Integer> scopes, byte[]... kvs) {
  WALKeyImpl key1 = new WALKeyImpl(new byte[0], TableName.valueOf("foo"),
    System.currentTimeMillis(), scopes);
  WALEdit edit1 = new WALEdit();

  for (byte[] kv : kvs) {
    edit1.add(new KeyValue(kv, kv, kv));
  }
  return new Entry(key1, edit1);
}
 
Example #19
Source File: Indexer.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Override
  public void preWALRestore(
          org.apache.hadoop.hbase.coprocessor.ObserverContext<? extends RegionCoprocessorEnvironment> ctx,
          org.apache.hadoop.hbase.client.RegionInfo info, org.apache.hadoop.hbase.wal.WALKey logKey, WALEdit logEdit)
          throws IOException {

    if (this.disabled) {
        return;
    }

  // TODO check the regions in transition. If the server on which the region lives is this one,
  // then we should rety that write later in postOpen.
  // we might be able to get even smarter here and pre-split the edits that are server-local
  // into their own recovered.edits file. This then lets us do a straightforward recovery of each
  // region (and more efficiently as we aren't writing quite as hectically from this one place).

    long start = EnvironmentEdgeManager.currentTimeMillis();
    try {
        /*
         * Basically, we let the index regions recover for a little while long before retrying in the
         * hopes they come up before the primary table finishes.
         */
        Collection<Pair<Mutation, byte[]>> indexUpdates = extractIndexUpdate(logEdit);
        recoveryWriter.writeAndHandleFailure(indexUpdates, true, ScanUtil.UNKNOWN_CLIENT_VERSION);
    } finally {
        long duration = EnvironmentEdgeManager.currentTimeMillis() - start;
        if (duration >= slowPreWALRestoreThreshold) {
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug(getCallTooSlowMessage("preWALRestore",
                        duration, slowPreWALRestoreThreshold));
            }
            metricSource.incrementNumSlowPreWALRestoreCalls();
        }
        metricSource.updatePreWALRestoreTime(duration);
    }
}
 
Example #20
Source File: TestHRegionServerBulkLoad.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void visitLogEntryBeforeWrite(WALKey logKey, WALEdit logEdit) {
  for (Cell cell : logEdit.getCells()) {
    KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
    for (Map.Entry entry : kv.toStringMap().entrySet()) {
      if (entry.getValue().equals(Bytes.toString(WALEdit.BULK_LOAD))) {
        found = true;
      }
    }
  }
}
 
Example #21
Source File: WALUtil.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Write the marker that a compaction has succeeded and is about to be committed. This provides
 * info to the HMaster to allow it to recover the compaction if this regionserver dies in the
 * middle. It also prevents the compaction from finishing if this regionserver has already lost
 * its lease on the log.
 * <p/>
 * This write is for internal use only. Not for external client consumption.
 * @param mvcc Used by WAL to get sequence Id for the waledit.
 */
public static WALKeyImpl writeCompactionMarker(WAL wal,
  NavigableMap<byte[], Integer> replicationScope, RegionInfo hri, final CompactionDescriptor c,
  MultiVersionConcurrencyControl mvcc) throws IOException {
  WALKeyImpl walKey =
    writeMarker(wal, replicationScope, hri, WALEdit.createCompaction(hri, c), mvcc, null);
  if (LOG.isTraceEnabled()) {
    LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c));
  }
  return walKey;
}
 
Example #22
Source File: AbstractTestCITimeout.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void preDelete(final ObserverContext<RegionCoprocessorEnvironment> e,
    final Delete delete, final WALEdit edit, final Durability durability) throws IOException {
  Threads.sleep(sleepTime.get());
  if (ct.incrementAndGet() == 1) {
    throw new IOException("first call I fail");
  }
}
 
Example #23
Source File: AsyncFSWAL.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
protected long append(RegionInfo hri, WALKeyImpl key, WALEdit edits, boolean inMemstore)
    throws IOException {
    if (markerEditOnly() && !edits.isMetaEdit()) {
      throw new IOException("WAL is closing, only marker edit is allowed");
    }
  long txid = stampSequenceIdAndPublishToRingBuffer(hri, key, edits, inMemstore,
    waitingConsumePayloads);
  if (shouldScheduleConsumer()) {
    consumeExecutor.execute(consumer);
  }
  return txid;
}
 
Example #24
Source File: TestHRegionReplayEvents.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Tests a case where we replay only a flush start marker, then the region is closed. This region
 * should not block indefinitely
 */
@Test
public void testOnlyReplayingFlushStartDoesNotHoldUpRegionClose() throws IOException {
  // load some data to primary and flush
  int start = 0;
  LOG.info("-- Writing some data to primary from " +  start + " to " + (start+100));
  putData(primaryRegion, Durability.SYNC_WAL, start, 100, cq, families);
  LOG.info("-- Flushing primary, creating 3 files for 3 stores");
  primaryRegion.flush(true);

  // now replay the edits and the flush marker
  reader = createWALReaderForPrimary();

  LOG.info("-- Replaying edits and flush events in secondary");
  while (true) {
    WAL.Entry entry = reader.next();
    if (entry == null) {
      break;
    }
    FlushDescriptor flushDesc
      = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
    if (flushDesc != null) {
      if (flushDesc.getAction() == FlushAction.START_FLUSH) {
        LOG.info("-- Replaying flush start in secondary");
        secondaryRegion.replayWALFlushStartMarker(flushDesc);
      } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) {
        LOG.info("-- NOT Replaying flush commit in secondary");
      }
    } else {
      replayEdit(secondaryRegion, entry);
    }
  }

  assertTrue(rss.getRegionServerAccounting().getGlobalMemStoreDataSize() > 0);
  // now close the region which should not cause hold because of un-committed flush
  secondaryRegion.close();

  // verify that the memstore size is back to what it was
  assertEquals(0, rss.getRegionServerAccounting().getGlobalMemStoreDataSize());
}
 
Example #25
Source File: SimpleRegionObserver.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void prePut(final ObserverContext<RegionCoprocessorEnvironment> c,
    final Put put, final WALEdit edit,
    final Durability durability) throws IOException {
  Map<byte[], List<Cell>> familyMap  = put.getFamilyCellMap();
  RegionCoprocessorEnvironment e = c.getEnvironment();
  assertNotNull(e);
  assertNotNull(e.getRegion());
  assertNotNull(familyMap);
  if (e.getRegion().getTableDescriptor().getTableName().equals(
      TestRegionObserverInterface.TEST_TABLE)) {
    List<Cell> cells = familyMap.get(TestRegionObserverInterface.A);
    assertNotNull(cells);
    assertNotNull(cells.get(0));
    Cell cell = cells.get(0);
    assertTrue(Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
      cell.getQualifierLength(), TestRegionObserverInterface.A, 0,
      TestRegionObserverInterface.A.length));
    cells = familyMap.get(TestRegionObserverInterface.B);
    assertNotNull(cells);
    assertNotNull(cells.get(0));
    cell = cells.get(0);
    assertTrue(Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
      cell.getQualifierLength(), TestRegionObserverInterface.B, 0,
      TestRegionObserverInterface.B.length));
    cells = familyMap.get(TestRegionObserverInterface.C);
    assertNotNull(cells);
    assertNotNull(cells.get(0));
    cell = cells.get(0);
    assertTrue(Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
      cell.getQualifierLength(), TestRegionObserverInterface.C, 0,
      TestRegionObserverInterface.C.length));
  }
  ctPrePut.incrementAndGet();
}
 
Example #26
Source File: SampleRegionWALCoprocessor.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void preWALWrite(ObserverContext<? extends WALCoprocessorEnvironment> env,
    RegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {
  // check table name matches or not.
  if (!Bytes.equals(info.getTable().toBytes(), this.tableName)) {
    return;
  }
  preWALWriteCalled = true;
  // here we're going to remove one keyvalue from the WALEdit, and add
  // another one to it.
  List<Cell> cells = logEdit.getCells();
  Cell deletedCell = null;
  for (Cell cell : cells) {
    // assume only one kv from the WALEdit matches.
    byte[] family = CellUtil.cloneFamily(cell);
    byte[] qulifier = CellUtil.cloneQualifier(cell);

    if (Arrays.equals(family, ignoredFamily) &&
        Arrays.equals(qulifier, ignoredQualifier)) {
      LOG.debug("Found the KeyValue from WALEdit which should be ignored.");
      deletedCell = cell;
    }
    if (Arrays.equals(family, changedFamily) &&
        Arrays.equals(qulifier, changedQualifier)) {
      LOG.debug("Found the KeyValue from WALEdit which should be changed.");
      cell.getValueArray()[cell.getValueOffset()] =
          (byte) (cell.getValueArray()[cell.getValueOffset()] + 1);
    }
  }
  if (null != row) {
    cells.add(new KeyValue(row, addedFamily, addedQualifier));
  }
  if (deletedCell != null) {
    LOG.debug("About to delete a KeyValue from WALEdit.");
    cells.remove(deletedCell);
  }
}
 
Example #27
Source File: MiniBatchOperationInProgress.java    From hbase with Apache License 2.0 5 votes vote down vote up
public MiniBatchOperationInProgress(T[] operations, OperationStatus[] retCodeDetails,
    WALEdit[] walEditsFromCoprocessors, int firstIndex, int lastIndexExclusive,
    int readyToWriteCount) {
  Preconditions.checkArgument(readyToWriteCount <= (lastIndexExclusive - firstIndex));
  this.operations = operations;
  this.retCodeDetails = retCodeDetails;
  this.walEditsFromCoprocessors = walEditsFromCoprocessors;
  this.firstIndex = firstIndex;
  this.lastIndexExclusive = lastIndexExclusive;
  this.readyToWriteCount = readyToWriteCount;
}
 
Example #28
Source File: TestWALObserver.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Coprocessors shouldn't get notice of empty waledits.
 */
@Test
public void testEmptyWALEditAreNotSeen() throws Exception {
  RegionInfo hri = createBasicHRegionInfo(Bytes.toString(TEST_TABLE));
  TableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE));
  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
  NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  for(byte[] fam : htd.getColumnFamilyNames()) {
    scopes.put(fam, 0);
  }
  WAL log = wals.getWAL(null);
  try {
    SampleRegionWALCoprocessor cp = getCoprocessor(log, SampleRegionWALCoprocessor.class);

    cp.setTestValues(TEST_TABLE, null, null, null, null, null, null, null);

    assertFalse(cp.isPreWALWriteCalled());
    assertFalse(cp.isPostWALWriteCalled());

    final long now = EnvironmentEdgeManager.currentTime();
    long txid = log.appendData(hri,
      new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(), now, mvcc, scopes),
      new WALEdit());
    log.sync(txid);

    assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPreWALWriteCalled());
    assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPostWALWriteCalled());
  } finally {
    log.close();
  }
}
 
Example #29
Source File: TestReplicationWithTags.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void prePut(final ObserverContext<RegionCoprocessorEnvironment> e, final Put put,
    final WALEdit edit, final Durability durability) throws IOException {
  byte[] attribute = put.getAttribute("visibility");
  byte[] cf = null;
  List<Cell> updatedCells = new ArrayList<>();
  if (attribute != null) {
    for (List<? extends Cell> edits : put.getFamilyCellMap().values()) {
      for (Cell cell : edits) {
        KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
        if (cf == null) {
          cf = CellUtil.cloneFamily(kv);
        }
        Tag tag = new ArrayBackedTag(TAG_TYPE, attribute);
        List<Tag> tagList = new ArrayList<>(1);
        tagList.add(tag);

        KeyValue newKV = new KeyValue(CellUtil.cloneRow(kv), 0, kv.getRowLength(),
            CellUtil.cloneFamily(kv), 0, kv.getFamilyLength(), CellUtil.cloneQualifier(kv), 0,
            kv.getQualifierLength(), kv.getTimestamp(),
            KeyValue.Type.codeToType(kv.getTypeByte()), CellUtil.cloneValue(kv), 0,
            kv.getValueLength(), tagList);
        ((List<Cell>) updatedCells).add(newKV);
      }
    }
    put.getFamilyCellMap().remove(cf);
    // Update the family map
    put.getFamilyCellMap().put(cf, updatedCells);
  }
}
 
Example #30
Source File: WALUtil.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Write a flush marker indicating a start / abort or a complete of a region flush
 * <p/>
 * This write is for internal use only. Not for external client consumption.
 */
public static WALKeyImpl writeFlushMarker(WAL wal, NavigableMap<byte[], Integer> replicationScope,
  RegionInfo hri, final FlushDescriptor f, boolean sync, MultiVersionConcurrencyControl mvcc)
  throws IOException {
  WALKeyImpl walKey = doFullMarkerAppendTransaction(wal, replicationScope, hri,
    WALEdit.createFlushWALEdit(hri, f), mvcc, null, sync);
  if (LOG.isTraceEnabled()) {
    LOG.trace("Appended flush marker " + TextFormat.shortDebugString(f));
  }
  return walKey;
}