Java Code Examples for org.apache.hadoop.hbase.wal.WAL#appendData()

The following examples show how to use org.apache.hadoop.hbase.wal.WAL#appendData() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestWALObserver.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void addWALEdits(final TableName tableName, final RegionInfo hri, final byte[] rowName,
    final byte[] family, final int count, EnvironmentEdge ee, final WAL wal,
    final NavigableMap<byte[], Integer> scopes, final MultiVersionConcurrencyControl mvcc)
    throws IOException {
  String familyStr = Bytes.toString(family);
  long txid = -1;
  for (int j = 0; j < count; j++) {
    byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
    byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
    WALEdit edit = new WALEdit();
    edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes));
    // uses WALKeyImpl instead of HLogKey on purpose. will only work for tests where we don't care
    // about legacy coprocessors
    txid = wal.appendData(hri,
      new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, ee.currentTime(), mvcc), edit);
  }
  if (-1 != txid) {
    wal.sync(txid);
  }
}
 
Example 2
Source File: TestWALObserver.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Coprocessors shouldn't get notice of empty waledits.
 */
@Test
public void testEmptyWALEditAreNotSeen() throws Exception {
  RegionInfo hri = createBasicHRegionInfo(Bytes.toString(TEST_TABLE));
  TableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE));
  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
  NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  for(byte[] fam : htd.getColumnFamilyNames()) {
    scopes.put(fam, 0);
  }
  WAL log = wals.getWAL(null);
  try {
    SampleRegionWALCoprocessor cp = getCoprocessor(log, SampleRegionWALCoprocessor.class);

    cp.setTestValues(TEST_TABLE, null, null, null, null, null, null, null);

    assertFalse(cp.isPreWALWriteCalled());
    assertFalse(cp.isPostWALWriteCalled());

    final long now = EnvironmentEdgeManager.currentTime();
    long txid = log.appendData(hri,
      new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(), now, mvcc, scopes),
      new WALEdit());
    log.sync(txid);

    assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPreWALWriteCalled());
    assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPostWALWriteCalled());
  } finally {
    log.close();
  }
}
 
Example 3
Source File: ProtobufLogTestHelper.java    From hbase with Apache License 2.0 5 votes vote down vote up
public static void doWrite(WAL wal, RegionInfo hri, TableName tableName, int columnCount,
    int recordCount, byte[] row, long timestamp, MultiVersionConcurrencyControl mvcc)
    throws IOException {
  for (int i = 0; i < recordCount; i++) {
    WAL.Entry entry = generateEdit(i, hri, tableName, row, columnCount, timestamp, mvcc);
    wal.appendData(hri, entry.getKey(), entry.getEdit());
  }
  wal.sync();
}
 
Example 4
Source File: AbstractTestFSWAL.java    From hbase with Apache License 2.0 5 votes vote down vote up
protected void addEdits(WAL log, RegionInfo hri, TableDescriptor htd, int times,
    MultiVersionConcurrencyControl mvcc, NavigableMap<byte[], Integer> scopes, String cf)
    throws IOException {
  final byte[] row = Bytes.toBytes(cf);
  for (int i = 0; i < times; i++) {
    long timestamp = System.currentTimeMillis();
    WALEdit cols = new WALEdit();
    cols.add(new KeyValue(row, row, row, timestamp, row));
    WALKeyImpl key = new WALKeyImpl(hri.getEncodedNameAsBytes(), htd.getTableName(),
        SequenceId.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE,
        HConstants.NO_NONCE, mvcc, scopes);
    log.appendData(hri, key, cols);
  }
  log.sync();
}
 
Example 5
Source File: AbstractTestWALReplay.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void addWALEdits(final TableName tableName, final RegionInfo hri, final byte[] rowName,
    final byte[] family, final int count, EnvironmentEdge ee, final WAL wal,
    final MultiVersionConcurrencyControl mvcc,
    NavigableMap<byte[], Integer> scopes) throws IOException {
  for (int j = 0; j < count; j++) {
    wal.appendData(hri, createWALKey(tableName, hri, mvcc, scopes),
      createWALEdit(rowName, family, ee, j));
  }
  wal.sync();
}
 
Example 6
Source File: TestWALActionsListener.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Add a bunch of dummy data and roll the logs every two insert. We
 * should end up with 10 rolled files (plus the roll called in
 * the constructor). Also test adding a listener while it's running.
 */
@Test
public void testActionListener() throws Exception {
  DummyWALActionsListener observer = new DummyWALActionsListener();
  final WALFactory wals = new WALFactory(conf, "testActionListener");
  wals.getWALProvider().addWALActionsListener(observer);
  DummyWALActionsListener laterobserver = new DummyWALActionsListener();
  RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf(SOME_BYTES))
      .setStartKey(SOME_BYTES).setEndKey(SOME_BYTES).build();
  final WAL wal = wals.getWAL(hri);
  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
  for (int i = 0; i < 20; i++) {
    byte[] b = Bytes.toBytes(i + "");
    KeyValue kv = new KeyValue(b, b, b);
    WALEdit edit = new WALEdit();
    edit.add(kv);
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    scopes.put(b, 0);
    long txid = wal.appendData(hri,
      new WALKeyImpl(hri.getEncodedNameAsBytes(), TableName.valueOf(b), 0, mvcc, scopes), edit);
    wal.sync(txid);
    if (i == 10) {
      wal.registerWALActionsListener(laterobserver);
    }
    if (i % 2 == 0) {
      wal.rollWriter();
    }
  }

  wal.close();

  assertEquals(11, observer.preLogRollCounter);
  assertEquals(11, observer.postLogRollCounter);
  assertEquals(5, laterobserver.preLogRollCounter);
  assertEquals(5, laterobserver.postLogRollCounter);
  assertEquals(1, observer.closedCount);
}
 
Example 7
Source File: TestReplicationSourceManager.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Test
public void testLogRoll() throws Exception {
  long baseline = 1000;
  long time = baseline;
  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
  KeyValue kv = new KeyValue(r1, f1, r1);
  WALEdit edit = new WALEdit();
  edit.add(kv);

  WALFactory wals =
    new WALFactory(utility.getConfiguration(), URLEncoder.encode("regionserver:60020", "UTF8"));
  ReplicationSourceManager replicationManager = replication.getReplicationManager();
  wals.getWALProvider()
    .addWALActionsListener(new ReplicationSourceWALActionListener(conf, replicationManager));
  final WAL wal = wals.getWAL(hri);
  manager.init();
  TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf("tableame"))
    .setColumnFamily(ColumnFamilyDescriptorBuilder.of(f1)).build();
  NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  for(byte[] fam : htd.getColumnFamilyNames()) {
    scopes.put(fam, 0);
  }
  // Testing normal log rolling every 20
  for(long i = 1; i < 101; i++) {
    if(i > 1 && i % 20 == 0) {
      wal.rollWriter();
    }
    LOG.info(Long.toString(i));
    final long txid = wal.appendData(hri,
      new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes),
      edit);
    wal.sync(txid);
  }

  // Simulate a rapid insert that's followed
  // by a report that's still not totally complete (missing last one)
  LOG.info(baseline + " and " + time);
  baseline += 101;
  time = baseline;
  LOG.info(baseline + " and " + time);

  for (int i = 0; i < 3; i++) {
    wal.appendData(hri,
      new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes),
      edit);
  }
  wal.sync();

  int logNumber = 0;
  for (Map.Entry<String, NavigableSet<String>> entry : manager.getWALs().get(slaveId)
    .entrySet()) {
    logNumber += entry.getValue().size();
  }
  assertEquals(6, logNumber);

  wal.rollWriter();

  ReplicationSourceInterface source = mock(ReplicationSourceInterface.class);
  when(source.getQueueId()).thenReturn("1");
  when(source.isRecovered()).thenReturn(false);
  when(source.isSyncReplication()).thenReturn(false);
  manager.logPositionAndCleanOldLogs(source,
    new WALEntryBatch(0, manager.getSources().get(0).getCurrentPath()));

  wal.appendData(hri,
    new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes),
    edit);
  wal.sync();

  assertEquals(1, manager.getWALs().size());


  // TODO Need a case with only 2 WALs and we only want to delete the first one
}
 
Example 8
Source File: TestLogRollAbort.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * Tests the case where a RegionServer enters a GC pause,
 * comes back online after the master declared it dead and started to split.
 * Want log rolling after a master split to fail. See HBASE-2312.
 */
@Test
public void testLogRollAfterSplitStart() throws IOException {
  LOG.info("Verify wal roll after split starts will fail.");
  String logName = ServerName.valueOf("testLogRollAfterSplitStart",
      16010, System.currentTimeMillis()).toString();
  Path thisTestsDir = new Path(HBASELOGDIR, AbstractFSWALProvider.getWALDirectoryName(logName));
  final WALFactory wals = new WALFactory(conf, logName);

  try {
    // put some entries in an WAL
    TableName tableName =
        TableName.valueOf(this.getClass().getName());
    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build();
    WAL log = wals.getWAL(regionInfo);
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);

    int total = 20;
    for (int i = 0; i < total; i++) {
      WALEdit kvs = new WALEdit();
      kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));
      NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
      scopes.put(Bytes.toBytes("column"), 0);
      log.appendData(regionInfo, new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName,
        System.currentTimeMillis(), mvcc, scopes), kvs);
    }
    // Send the data to HDFS datanodes and close the HDFS writer
    log.sync();
    ((AbstractFSWAL<?>) log).replaceWriter(((FSHLog)log).getOldPath(), null, null);

    // code taken from MasterFileSystem.getLogDirs(), which is called from
    // MasterFileSystem.splitLog() handles RS shutdowns (as observed by the splitting process)
    // rename the directory so a rogue RS doesn't create more WALs
    Path rsSplitDir = thisTestsDir.suffix(AbstractFSWALProvider.SPLITTING_EXT);
    if (!fs.rename(thisTestsDir, rsSplitDir)) {
      throw new IOException("Failed fs.rename for log split: " + thisTestsDir);
    }
    LOG.debug("Renamed region directory: " + rsSplitDir);

    LOG.debug("Processing the old log files.");
    WALSplitter.split(HBASELOGDIR, rsSplitDir, OLDLOGDIR, fs, conf, wals);

    LOG.debug("Trying to roll the WAL.");
    try {
      log.rollWriter();
      Assert.fail("rollWriter() did not throw any exception.");
    } catch (IOException ioe) {
      if (ioe.getCause() instanceof FileNotFoundException) {
        LOG.info("Got the expected exception: ", ioe.getCause());
      } else {
        Assert.fail("Unexpected exception: " + ioe);
      }
    }
  } finally {
    wals.close();
    if (fs.exists(thisTestsDir)) {
      fs.delete(thisTestsDir, true);
    }
  }
}