Java Code Examples for org.apache.hadoop.hbase.wal.WAL#Reader

The following examples show how to use org.apache.hadoop.hbase.wal.WAL#Reader . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SerialReplicationTestBase.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected final void checkOrder(int expectedEntries) throws IOException {
  try (WAL.Reader reader =
    WALFactory.createReader(UTIL.getTestFileSystem(), logPath, UTIL.getConfiguration())) {
    long seqId = -1L;
    int count = 0;
    for (Entry entry;;) {
      entry = reader.next();
      if (entry == null) {
        break;
      }
      assertTrue(
        "Sequence id go backwards from " + seqId + " to " + entry.getKey().getSequenceId(),
        entry.getKey().getSequenceId() >= seqId);
      seqId = entry.getKey().getSequenceId();
      count++;
    }
    assertEquals(expectedEntries, count);
  }
}
 
Example 2
Source File: TestSequenceIdMonotonicallyIncreasing.java    From hbase with Apache License 2.0 6 votes vote down vote up
private long getMaxSeqId(HRegionServer rs, RegionInfo region) throws IOException {
  Path walFile = ((AbstractFSWAL<?>) rs.getWAL(null)).getCurrentFileName();
  long maxSeqId = -1L;
  try (WAL.Reader reader =
    WALFactory.createReader(UTIL.getTestFileSystem(), walFile, UTIL.getConfiguration())) {
    for (;;) {
      WAL.Entry entry = reader.next();
      if (entry == null) {
        break;
      }
      if (Bytes.equals(region.getEncodedNameAsBytes(), entry.getKey().getEncodedRegionName())) {
        maxSeqId = Math.max(maxSeqId, entry.getKey().getSequenceId());
      }
    }
  }
  return maxSeqId;
}
 
Example 3
Source File: Compressor.java    From hbase with Apache License 2.0 5 votes vote down vote up
private static void transformFile(Path input, Path output)
    throws IOException {
  Configuration conf = HBaseConfiguration.create();

  FileSystem inFS = input.getFileSystem(conf);
  FileSystem outFS = output.getFileSystem(conf);

  WAL.Reader in = WALFactory.createReaderIgnoreCustomClass(inFS, input, conf);
  WALProvider.Writer out = null;

  try {
    if (!(in instanceof ReaderBase)) {
      System.err.println("Cannot proceed, invalid reader type: " + in.getClass().getName());
      return;
    }
    boolean compress = ((ReaderBase)in).hasCompression();
    conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, !compress);
    out = WALFactory.createWALWriter(outFS, output, conf);

    WAL.Entry e = null;
    while ((e = in.next()) != null) out.append(e);
  } finally {
    in.close();
    if (out != null) {
      out.close();
      out = null;
    }
  }
}
 
Example 4
Source File: TestReplicationSource.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Sanity check that we can move logs around while we are reading
 * from them. Should this test fail, ReplicationSource would have a hard
 * time reading logs that are being archived.
 */
@Test
public void testLogMoving() throws Exception{
  Path logPath = new Path(logDir, "log");
  if (!FS.exists(logDir)) FS.mkdirs(logDir);
  if (!FS.exists(oldLogDir)) FS.mkdirs(oldLogDir);
  WALProvider.Writer writer = WALFactory.createWALWriter(FS, logPath,
      TEST_UTIL.getConfiguration());
  for(int i = 0; i < 3; i++) {
    byte[] b = Bytes.toBytes(Integer.toString(i));
    KeyValue kv = new KeyValue(b,b,b);
    WALEdit edit = new WALEdit();
    edit.add(kv);
    WALKeyImpl key = new WALKeyImpl(b, TableName.valueOf(b), 0, 0,
        HConstants.DEFAULT_CLUSTER_ID);
    writer.append(new WAL.Entry(key, edit));
    writer.sync(false);
  }
  writer.close();

  WAL.Reader reader = WALFactory.createReader(FS, logPath, TEST_UTIL.getConfiguration());
  WAL.Entry entry = reader.next();
  assertNotNull(entry);

  Path oldLogPath = new Path(oldLogDir, "log");
  FS.rename(logPath, oldLogPath);

  entry = reader.next();
  assertNotNull(entry);

  entry = reader.next();
  entry = reader.next();

  assertNull(entry);
  reader.close();
}
 
Example 5
Source File: AbstractTestDLS.java    From hbase with Apache License 2.0 5 votes vote down vote up
private int countWAL(Path log, FileSystem fs, Configuration conf) throws IOException {
  int count = 0;
  try (WAL.Reader in = WALFactory.createReader(fs, log, conf)) {
    WAL.Entry e;
    while ((e = in.next()) != null) {
      if (!WALEdit.isMetaEditFamily(e.getEdit().getCells().get(0))) {
        count++;
      }
    }
  }
  return count;
}
 
Example 6
Source File: TestDurability.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void verifyWALCount(WALFactory wals, WAL log, int expected) throws Exception {
  Path walPath = AbstractFSWALProvider.getCurrentFileName(log);
  WAL.Reader reader = wals.createReader(FS, walPath);
  int count = 0;
  WAL.Entry entry = new WAL.Entry();
  while (reader.next(entry) != null) {
    count++;
  }
  reader.close();
  assertEquals(expected, count);
}
 
Example 7
Source File: TestWALMonotonicallyIncreasingSeqId.java    From hbase with Apache License 2.0 5 votes vote down vote up
private WAL.Reader createReader(Path logPath, Path oldWalsDir) throws IOException {
  try {
    return wals.createReader(fileSystem, logPath);
  } catch (IOException e) {
    return wals.createReader(fileSystem, new Path(oldWalsDir, logPath.getName()));
  }
}
 
Example 8
Source File: TestWALMonotonicallyIncreasingSeqId.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testWALMonotonicallyIncreasingSeqId() throws Exception {
  List<Thread> putThreads = new ArrayList<>();
  for (int i = 0; i < 1; i++) {
    putThreads.add(new PutThread(region));
  }
  IncThread incThread = new IncThread(region);
  for (int i = 0; i < 1; i++) {
    putThreads.get(i).start();
  }
  incThread.start();
  incThread.join();

  Path logPath = ((AbstractFSWAL<?>) region.getWAL()).getCurrentFileName();
  region.getWAL().rollWriter();
  Thread.sleep(10);
  Path hbaseDir = new Path(walConf.get(HConstants.HBASE_DIR));
  Path oldWalsDir = new Path(hbaseDir, HConstants.HREGION_OLDLOGDIR_NAME);
  try (WAL.Reader reader = createReader(logPath, oldWalsDir)) {
    long currentMaxSeqid = 0;
    for (WAL.Entry e; (e = reader.next()) != null;) {
      if (!WALEdit.isMetaEditFamily(e.getEdit().getCells().get(0))) {
        long currentSeqid = e.getKey().getSequenceId();
        if (currentSeqid > currentMaxSeqid) {
          currentMaxSeqid = currentSeqid;
        } else {
          fail("Current max Seqid is " + currentMaxSeqid +
            ", but the next seqid in wal is smaller:" + currentSeqid);
        }
      }
    }
  }
}
 
Example 9
Source File: WALProcedurePrettyPrinter.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
protected int doWork() throws Exception {
  Path path = new Path(file);
  FileSystem fs = path.getFileSystem(conf);
  try (WAL.Reader reader = WALFactory.createReader(fs, path, conf)) {
    for (;;) {
      WAL.Entry entry = reader.next();
      if (entry == null) {
        return 0;
      }
      WALKey key = entry.getKey();
      WALEdit edit = entry.getEdit();
      long sequenceId = key.getSequenceId();
      long writeTime = key.getWriteTime();
      out.println(
        String.format(KEY_TMPL, sequenceId, FORMATTER.format(Instant.ofEpochMilli(writeTime))));
      for (Cell cell : edit.getCells()) {
        Map<String, Object> op = WALPrettyPrinter.toStringMap(cell);
        if (!Bytes.equals(PROC_FAMILY, 0, PROC_FAMILY.length, cell.getFamilyArray(),
          cell.getFamilyOffset(), cell.getFamilyLength())) {
          // We could have cells other than procedure edits, for example, a flush marker
          WALPrettyPrinter.printCell(out, op, false);
          continue;
        }
        long procId = Bytes.toLong(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
        out.println("pid=" + procId + ", type=" + op.get("type") + ", column=" +
          op.get("family") + ":" + op.get("qualifier"));
        if (cell.getType() == Cell.Type.Put) {
          if (cell.getValueLength() > 0) {
            // should be a normal put
            Procedure<?> proc =
              ProcedureUtil.convertToProcedure(ProcedureProtos.Procedure.parser()
                .parseFrom(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
            out.println("\t" + proc.toStringDetails());
          } else {
            // should be a 'delete' put
            out.println("\tmark deleted");
          }
        }
        out.println("cell total size sum: " + cell.heapSize());
      }
      out.println("edit heap size: " + edit.heapSize());
      out.println("position: " + reader.getPosition());
    }
  }
}
 
Example 10
Source File: TestSerialReplication.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Test
public void testRegionSplit() throws Exception {
  TableName tableName = createTable();
  try (Table table = UTIL.getConnection().getTable(tableName)) {
    for (int i = 0; i < 100; i++) {
      table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
    }
  }
  UTIL.flush(tableName);
  RegionInfo region = UTIL.getAdmin().getRegions(tableName).get(0);
  UTIL.getAdmin().splitRegionAsync(region.getEncodedNameAsBytes(), Bytes.toBytes(50)).get(30,
    TimeUnit.SECONDS);
  UTIL.waitUntilNoRegionsInTransition(30000);
  List<RegionInfo> regions = UTIL.getAdmin().getRegions(tableName);
  assertEquals(2, regions.size());
  try (Table table = UTIL.getConnection().getTable(tableName)) {
    for (int i = 0; i < 100; i++) {
      table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
    }
  }
  enablePeerAndWaitUntilReplicationDone(200);
  Map<String, Long> regionsToSeqId = new HashMap<>();
  regionsToSeqId.put(region.getEncodedName(), -1L);
  regions.stream().map(RegionInfo::getEncodedName).forEach(n -> regionsToSeqId.put(n, -1L));
  try (WAL.Reader reader =
    WALFactory.createReader(UTIL.getTestFileSystem(), logPath, UTIL.getConfiguration())) {
    int count = 0;
    for (Entry entry;;) {
      entry = reader.next();
      if (entry == null) {
        break;
      }
      String encodedName = Bytes.toString(entry.getKey().getEncodedRegionName());
      Long seqId = regionsToSeqId.get(encodedName);
      assertNotNull(
        "Unexcepted entry " + entry + ", expected regions " + region + ", or " + regions, seqId);
      assertTrue("Sequence id go backwards from " + seqId + " to " +
        entry.getKey().getSequenceId() + " for " + encodedName,
        entry.getKey().getSequenceId() >= seqId.longValue());
      if (count < 100) {
        assertEquals(encodedName + " is pushed before parent " + region.getEncodedName(),
          region.getEncodedName(), encodedName);
      } else {
        assertNotEquals(region.getEncodedName(), encodedName);
      }
      count++;
    }
    assertEquals(200, count);
  }
}
 
Example 11
Source File: TestSerialReplication.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Test
public void testRegionMerge() throws Exception {
  byte[] splitKey = Bytes.toBytes(50);
  TableName tableName = TableName.valueOf(name.getMethodName());
  UTIL.getAdmin().createTable(
    TableDescriptorBuilder.newBuilder(tableName)
      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(CF)
        .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build())
      .build(),
    new byte[][] { splitKey });
  UTIL.waitTableAvailable(tableName);
  try (Table table = UTIL.getConnection().getTable(tableName)) {
    for (int i = 0; i < 100; i++) {
      table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
    }
  }
  List<RegionInfo> regions = UTIL.getAdmin().getRegions(tableName);
  UTIL.getAdmin()
    .mergeRegionsAsync(
      regions.stream().map(RegionInfo::getEncodedNameAsBytes).toArray(byte[][]::new), false)
    .get(30, TimeUnit.SECONDS);
  UTIL.waitUntilNoRegionsInTransition(30000);
  List<RegionInfo> regionsAfterMerge = UTIL.getAdmin().getRegions(tableName);
  assertEquals(1, regionsAfterMerge.size());
  try (Table table = UTIL.getConnection().getTable(tableName)) {
    for (int i = 0; i < 100; i++) {
      table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
    }
  }
  enablePeerAndWaitUntilReplicationDone(200);
  Map<String, Long> regionsToSeqId = new HashMap<>();
  RegionInfo region = regionsAfterMerge.get(0);
  regionsToSeqId.put(region.getEncodedName(), -1L);
  regions.stream().map(RegionInfo::getEncodedName).forEach(n -> regionsToSeqId.put(n, -1L));
  try (WAL.Reader reader =
    WALFactory.createReader(UTIL.getTestFileSystem(), logPath, UTIL.getConfiguration())) {
    int count = 0;
    for (Entry entry;;) {
      entry = reader.next();
      if (entry == null) {
        break;
      }
      String encodedName = Bytes.toString(entry.getKey().getEncodedRegionName());
      Long seqId = regionsToSeqId.get(encodedName);
      assertNotNull(
        "Unexcepted entry " + entry + ", expected regions " + region + ", or " + regions, seqId);
      assertTrue("Sequence id go backwards from " + seqId + " to " +
        entry.getKey().getSequenceId() + " for " + encodedName,
        entry.getKey().getSequenceId() >= seqId.longValue());
      if (count < 100) {
        assertNotEquals(
          encodedName + " is pushed before parents " +
            regions.stream().map(RegionInfo::getEncodedName).collect(Collectors.joining(" and ")),
          region.getEncodedName(), encodedName);
      } else {
        assertEquals(region.getEncodedName(), encodedName);
      }
      count++;
    }
    assertEquals(200, count);
  }
}
 
Example 12
Source File: TestRaceWhenCreatingReplicationSource.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Test
public void testRace() throws Exception {
  UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {

    @Override
    public boolean evaluate() throws Exception {
      for (RegionServerThread t : UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
        ReplicationSource source =
          (ReplicationSource) ((Replication) t.getRegionServer().getReplicationSourceService())
            .getReplicationManager().getSource(PEER_ID);
        if (source == null || source.getReplicationEndpoint() == null) {
          return false;
        }
      }
      return true;
    }

    @Override
    public String explainFailure() throws Exception {
      return "Replication source has not been initialized yet";
    }
  });
  UTIL.getAdmin().createTable(
    TableDescriptorBuilder.newBuilder(TABLE_NAME).setColumnFamily(ColumnFamilyDescriptorBuilder
      .newBuilder(CF).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()).build());
  UTIL.waitTableAvailable(TABLE_NAME);
  try (Table table = UTIL.getConnection().getTable(TABLE_NAME)) {
    table.put(new Put(Bytes.toBytes(1)).addColumn(CF, CQ, Bytes.toBytes(1)));
  }
  NULL_UUID = false;
  UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {

    @Override
    public boolean evaluate() throws Exception {
      try (WAL.Reader reader = WALFactory.createReader(FS, LOG_PATH, UTIL.getConfiguration())) {
        return reader.next() != null;
      } catch (IOException e) {
        return false;
      }
    }

    @Override
    public String explainFailure() throws Exception {
      return "Replication has not catched up";
    }
  });
  try (WAL.Reader reader = WALFactory.createReader(FS, LOG_PATH, UTIL.getConfiguration())) {
    Cell cell = reader.next().getEdit().getCells().get(0);
    assertEquals(1, Bytes.toInt(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
    assertArrayEquals(CF, CellUtil.cloneFamily(cell));
    assertArrayEquals(CQ, CellUtil.cloneQualifier(cell));
    assertEquals(1,
      Bytes.toInt(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
  }
}
 
Example 13
Source File: TestRecoveredEdits.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * @param fs
 * @param conf
 * @param edits
 * @param region
 * @return Return how many edits seen.
 * @throws IOException
 */
private int verifyAllEditsMadeItIn(final FileSystem fs, final Configuration conf,
    final Path edits, final HRegion region) throws IOException {
  int count = 0;
  // Read all cells from recover edits
  List<Cell> walCells = new ArrayList<>();
  try (WAL.Reader reader = WALFactory.createReader(fs, edits, conf)) {
    WAL.Entry entry;
    while ((entry = reader.next()) != null) {
      WALKey key = entry.getKey();
      WALEdit val = entry.getEdit();
      count++;
      // Check this edit is for this region.
      if (!Bytes.equals(key.getEncodedRegionName(),
          region.getRegionInfo().getEncodedNameAsBytes())) {
        continue;
      }
      Cell previous = null;
      for (Cell cell : val.getCells()) {
        if (WALEdit.isMetaEditFamily(cell)) {
          continue;
        }
        if (previous != null && CellComparatorImpl.COMPARATOR.compareRows(previous, cell) == 0) {
          continue;
        }
        previous = cell;
        walCells.add(cell);
      }
    }
  }

  // Read all cells from region
  List<Cell> regionCells = new ArrayList<>();
  try (RegionScanner scanner = region.getScanner(new Scan())) {
    List<Cell> tmpCells;
    do {
      tmpCells = new ArrayList<>();
      scanner.nextRaw(tmpCells);
      regionCells.addAll(tmpCells);
    } while (!tmpCells.isEmpty());
  }

  Collections.sort(walCells, CellComparatorImpl.COMPARATOR);
  int found = 0;
  for (int i = 0, j = 0; i < walCells.size() && j < regionCells.size(); ) {
    int compareResult = PrivateCellUtil
        .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, walCells.get(i),
            regionCells.get(j));
    if (compareResult == 0) {
      i++;
      j++;
      found++;
    } else if (compareResult > 0) {
      j++;
    } else {
      i++;
    }
  }
  assertEquals("Only found " + found + " cells in region, but there are " + walCells.size() +
      " cells in recover edits", found, walCells.size());
  return count;
}
 
Example 14
Source File: TestHRegionReplayEvents.java    From hbase with Apache License 2.0 4 votes vote down vote up
WAL.Reader createWALReaderForPrimary() throws FileNotFoundException, IOException {
  return WALFactory.createReader(TEST_UTIL.getTestFileSystem(),
    AbstractFSWALProvider.getCurrentFileName(walPrimary),
    TEST_UTIL.getConfiguration());
}