Java Code Examples for org.apache.hadoop.hbase.wal.WALFactory#createReader()
The following examples show how to use
org.apache.hadoop.hbase.wal.WALFactory#createReader() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestSyncReplicationActive.java From hbase with Apache License 2.0 | 6 votes |
private void verifyNoClusterIdInRemoteLog(HBaseTestingUtility utility, Path remoteDir, String peerId) throws Exception { FileSystem fs2 = utility.getTestFileSystem(); FileStatus[] files = fs2.listStatus(new Path(remoteDir, peerId)); Assert.assertTrue(files.length > 0); for (FileStatus file : files) { try ( Reader reader = WALFactory.createReader(fs2, file.getPath(), utility.getConfiguration())) { Entry entry = reader.next(); Assert.assertTrue(entry != null); while (entry != null) { Assert.assertEquals(entry.getKey().getClusterIds().size(), 0); entry = reader.next(); } } } }
Example 2
Source File: SerialReplicationTestBase.java From hbase with Apache License 2.0 | 6 votes |
protected final void checkOrder(int expectedEntries) throws IOException { try (WAL.Reader reader = WALFactory.createReader(UTIL.getTestFileSystem(), logPath, UTIL.getConfiguration())) { long seqId = -1L; int count = 0; for (Entry entry;;) { entry = reader.next(); if (entry == null) { break; } assertTrue( "Sequence id go backwards from " + seqId + " to " + entry.getKey().getSequenceId(), entry.getKey().getSequenceId() >= seqId); seqId = entry.getKey().getSequenceId(); count++; } assertEquals(expectedEntries, count); } }
Example 3
Source File: TestSequenceIdMonotonicallyIncreasing.java From hbase with Apache License 2.0 | 6 votes |
private long getMaxSeqId(HRegionServer rs, RegionInfo region) throws IOException { Path walFile = ((AbstractFSWAL<?>) rs.getWAL(null)).getCurrentFileName(); long maxSeqId = -1L; try (WAL.Reader reader = WALFactory.createReader(UTIL.getTestFileSystem(), walFile, UTIL.getConfiguration())) { for (;;) { WAL.Entry entry = reader.next(); if (entry == null) { break; } if (Bytes.equals(region.getEncodedNameAsBytes(), entry.getKey().getEncodedRegionName())) { maxSeqId = Math.max(maxSeqId, entry.getKey().getSequenceId()); } } } return maxSeqId; }
Example 4
Source File: WALEntryStream.java From hbase with Apache License 2.0 | 5 votes |
private void openReader(Path path) throws IOException { try { // Detect if this is a new file, if so get a new reader else // reset the current reader so that we see the new data if (reader == null || !getCurrentPath().equals(path)) { closeReader(); reader = WALFactory.createReader(fs, path, conf); seek(); setCurrentPath(path); } else { resetReader(); } } catch (FileNotFoundException fnfe) { handleFileNotFound(path, fnfe); } catch (RemoteException re) { IOException ioe = re.unwrapRemoteException(FileNotFoundException.class); if (!(ioe instanceof FileNotFoundException)) throw ioe; handleFileNotFound(path, (FileNotFoundException)ioe); } catch (LeaseNotRecoveredException lnre) { // HBASE-15019 the WAL was not closed due to some hiccup. LOG.warn("Try to recover the WAL lease " + currentPath, lnre); recoverLease(conf, currentPath); reader = null; } catch (NullPointerException npe) { // Workaround for race condition in HDFS-4380 // which throws a NPE if we open a file before any data node has the most recent block // Just sleep and retry. Will require re-reading compressed WALs for compressionContext. LOG.warn("Got NPE opening reader, will retry."); reader = null; } }
Example 5
Source File: ReplaySyncReplicationWALCallable.java From hbase with Apache License 2.0 | 5 votes |
private Reader getReader(String wal) throws IOException { Path path = new Path(rs.getWALRootDir(), wal); long length = rs.getWALFileSystem().getFileStatus(path).getLen(); try { RecoverLeaseFSUtils.recoverFileLease(fs, path, conf); return WALFactory.createReader(rs.getWALFileSystem(), path, rs.getConfiguration()); } catch (EOFException e) { if (length <= 0) { LOG.warn("File is empty. Could not open {} for reading because {}", path, e); return null; } throw e; } }
Example 6
Source File: TestReplicationSource.java From hbase with Apache License 2.0 | 5 votes |
/** * Sanity check that we can move logs around while we are reading * from them. Should this test fail, ReplicationSource would have a hard * time reading logs that are being archived. */ @Test public void testLogMoving() throws Exception{ Path logPath = new Path(logDir, "log"); if (!FS.exists(logDir)) FS.mkdirs(logDir); if (!FS.exists(oldLogDir)) FS.mkdirs(oldLogDir); WALProvider.Writer writer = WALFactory.createWALWriter(FS, logPath, TEST_UTIL.getConfiguration()); for(int i = 0; i < 3; i++) { byte[] b = Bytes.toBytes(Integer.toString(i)); KeyValue kv = new KeyValue(b,b,b); WALEdit edit = new WALEdit(); edit.add(kv); WALKeyImpl key = new WALKeyImpl(b, TableName.valueOf(b), 0, 0, HConstants.DEFAULT_CLUSTER_ID); writer.append(new WAL.Entry(key, edit)); writer.sync(false); } writer.close(); WAL.Reader reader = WALFactory.createReader(FS, logPath, TEST_UTIL.getConfiguration()); WAL.Entry entry = reader.next(); assertNotNull(entry); Path oldLogPath = new Path(oldLogDir, "log"); FS.rename(logPath, oldLogPath); entry = reader.next(); assertNotNull(entry); entry = reader.next(); entry = reader.next(); assertNull(entry); reader.close(); }
Example 7
Source File: AbstractTestDLS.java From hbase with Apache License 2.0 | 5 votes |
private int countWAL(Path log, FileSystem fs, Configuration conf) throws IOException { int count = 0; try (WAL.Reader in = WALFactory.createReader(fs, log, conf)) { WAL.Entry e; while ((e = in.next()) != null) { if (!WALEdit.isMetaEditFamily(e.getEdit().getCells().get(0))) { count++; } } } return count; }
Example 8
Source File: TestDurability.java From hbase with Apache License 2.0 | 5 votes |
private void verifyWALCount(WALFactory wals, WAL log, int expected) throws Exception { Path walPath = AbstractFSWALProvider.getCurrentFileName(log); WAL.Reader reader = wals.createReader(FS, walPath); int count = 0; WAL.Entry entry = new WAL.Entry(); while (reader.next(entry) != null) { count++; } reader.close(); assertEquals(expected, count); }
Example 9
Source File: WALProcedurePrettyPrinter.java From hbase with Apache License 2.0 | 4 votes |
@Override protected int doWork() throws Exception { Path path = new Path(file); FileSystem fs = path.getFileSystem(conf); try (WAL.Reader reader = WALFactory.createReader(fs, path, conf)) { for (;;) { WAL.Entry entry = reader.next(); if (entry == null) { return 0; } WALKey key = entry.getKey(); WALEdit edit = entry.getEdit(); long sequenceId = key.getSequenceId(); long writeTime = key.getWriteTime(); out.println( String.format(KEY_TMPL, sequenceId, FORMATTER.format(Instant.ofEpochMilli(writeTime)))); for (Cell cell : edit.getCells()) { Map<String, Object> op = WALPrettyPrinter.toStringMap(cell); if (!Bytes.equals(PROC_FAMILY, 0, PROC_FAMILY.length, cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength())) { // We could have cells other than procedure edits, for example, a flush marker WALPrettyPrinter.printCell(out, op, false); continue; } long procId = Bytes.toLong(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); out.println("pid=" + procId + ", type=" + op.get("type") + ", column=" + op.get("family") + ":" + op.get("qualifier")); if (cell.getType() == Cell.Type.Put) { if (cell.getValueLength() > 0) { // should be a normal put Procedure<?> proc = ProcedureUtil.convertToProcedure(ProcedureProtos.Procedure.parser() .parseFrom(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); out.println("\t" + proc.toStringDetails()); } else { // should be a 'delete' put out.println("\tmark deleted"); } } out.println("cell total size sum: " + cell.heapSize()); } out.println("edit heap size: " + edit.heapSize()); out.println("position: " + reader.getPosition()); } } }
Example 10
Source File: TestSerialReplication.java From hbase with Apache License 2.0 | 4 votes |
@Test public void testRegionSplit() throws Exception { TableName tableName = createTable(); try (Table table = UTIL.getConnection().getTable(tableName)) { for (int i = 0; i < 100; i++) { table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); } } UTIL.flush(tableName); RegionInfo region = UTIL.getAdmin().getRegions(tableName).get(0); UTIL.getAdmin().splitRegionAsync(region.getEncodedNameAsBytes(), Bytes.toBytes(50)).get(30, TimeUnit.SECONDS); UTIL.waitUntilNoRegionsInTransition(30000); List<RegionInfo> regions = UTIL.getAdmin().getRegions(tableName); assertEquals(2, regions.size()); try (Table table = UTIL.getConnection().getTable(tableName)) { for (int i = 0; i < 100; i++) { table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); } } enablePeerAndWaitUntilReplicationDone(200); Map<String, Long> regionsToSeqId = new HashMap<>(); regionsToSeqId.put(region.getEncodedName(), -1L); regions.stream().map(RegionInfo::getEncodedName).forEach(n -> regionsToSeqId.put(n, -1L)); try (WAL.Reader reader = WALFactory.createReader(UTIL.getTestFileSystem(), logPath, UTIL.getConfiguration())) { int count = 0; for (Entry entry;;) { entry = reader.next(); if (entry == null) { break; } String encodedName = Bytes.toString(entry.getKey().getEncodedRegionName()); Long seqId = regionsToSeqId.get(encodedName); assertNotNull( "Unexcepted entry " + entry + ", expected regions " + region + ", or " + regions, seqId); assertTrue("Sequence id go backwards from " + seqId + " to " + entry.getKey().getSequenceId() + " for " + encodedName, entry.getKey().getSequenceId() >= seqId.longValue()); if (count < 100) { assertEquals(encodedName + " is pushed before parent " + region.getEncodedName(), region.getEncodedName(), encodedName); } else { assertNotEquals(region.getEncodedName(), encodedName); } count++; } assertEquals(200, count); } }
Example 11
Source File: TestSerialReplication.java From hbase with Apache License 2.0 | 4 votes |
@Test public void testRegionMerge() throws Exception { byte[] splitKey = Bytes.toBytes(50); TableName tableName = TableName.valueOf(name.getMethodName()); UTIL.getAdmin().createTable( TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(CF) .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()) .build(), new byte[][] { splitKey }); UTIL.waitTableAvailable(tableName); try (Table table = UTIL.getConnection().getTable(tableName)) { for (int i = 0; i < 100; i++) { table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); } } List<RegionInfo> regions = UTIL.getAdmin().getRegions(tableName); UTIL.getAdmin() .mergeRegionsAsync( regions.stream().map(RegionInfo::getEncodedNameAsBytes).toArray(byte[][]::new), false) .get(30, TimeUnit.SECONDS); UTIL.waitUntilNoRegionsInTransition(30000); List<RegionInfo> regionsAfterMerge = UTIL.getAdmin().getRegions(tableName); assertEquals(1, regionsAfterMerge.size()); try (Table table = UTIL.getConnection().getTable(tableName)) { for (int i = 0; i < 100; i++) { table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); } } enablePeerAndWaitUntilReplicationDone(200); Map<String, Long> regionsToSeqId = new HashMap<>(); RegionInfo region = regionsAfterMerge.get(0); regionsToSeqId.put(region.getEncodedName(), -1L); regions.stream().map(RegionInfo::getEncodedName).forEach(n -> regionsToSeqId.put(n, -1L)); try (WAL.Reader reader = WALFactory.createReader(UTIL.getTestFileSystem(), logPath, UTIL.getConfiguration())) { int count = 0; for (Entry entry;;) { entry = reader.next(); if (entry == null) { break; } String encodedName = Bytes.toString(entry.getKey().getEncodedRegionName()); Long seqId = regionsToSeqId.get(encodedName); assertNotNull( "Unexcepted entry " + entry + ", expected regions " + region + ", or " + regions, seqId); assertTrue("Sequence id go backwards from " + seqId + " to " + entry.getKey().getSequenceId() + " for " + encodedName, entry.getKey().getSequenceId() >= seqId.longValue()); if (count < 100) { assertNotEquals( encodedName + " is pushed before parents " + regions.stream().map(RegionInfo::getEncodedName).collect(Collectors.joining(" and ")), region.getEncodedName(), encodedName); } else { assertEquals(region.getEncodedName(), encodedName); } count++; } assertEquals(200, count); } }
Example 12
Source File: TestRaceWhenCreatingReplicationSource.java From hbase with Apache License 2.0 | 4 votes |
@Test public void testRace() throws Exception { UTIL.waitFor(30000, new ExplainingPredicate<Exception>() { @Override public boolean evaluate() throws Exception { for (RegionServerThread t : UTIL.getMiniHBaseCluster().getRegionServerThreads()) { ReplicationSource source = (ReplicationSource) ((Replication) t.getRegionServer().getReplicationSourceService()) .getReplicationManager().getSource(PEER_ID); if (source == null || source.getReplicationEndpoint() == null) { return false; } } return true; } @Override public String explainFailure() throws Exception { return "Replication source has not been initialized yet"; } }); UTIL.getAdmin().createTable( TableDescriptorBuilder.newBuilder(TABLE_NAME).setColumnFamily(ColumnFamilyDescriptorBuilder .newBuilder(CF).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()).build()); UTIL.waitTableAvailable(TABLE_NAME); try (Table table = UTIL.getConnection().getTable(TABLE_NAME)) { table.put(new Put(Bytes.toBytes(1)).addColumn(CF, CQ, Bytes.toBytes(1))); } NULL_UUID = false; UTIL.waitFor(30000, new ExplainingPredicate<Exception>() { @Override public boolean evaluate() throws Exception { try (WAL.Reader reader = WALFactory.createReader(FS, LOG_PATH, UTIL.getConfiguration())) { return reader.next() != null; } catch (IOException e) { return false; } } @Override public String explainFailure() throws Exception { return "Replication has not catched up"; } }); try (WAL.Reader reader = WALFactory.createReader(FS, LOG_PATH, UTIL.getConfiguration())) { Cell cell = reader.next().getEdit().getCells().get(0); assertEquals(1, Bytes.toInt(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())); assertArrayEquals(CF, CellUtil.cloneFamily(cell)); assertArrayEquals(CQ, CellUtil.cloneQualifier(cell)); assertEquals(1, Bytes.toInt(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); } }
Example 13
Source File: TestRecoveredEdits.java From hbase with Apache License 2.0 | 4 votes |
/** * @param fs * @param conf * @param edits * @param region * @return Return how many edits seen. * @throws IOException */ private int verifyAllEditsMadeItIn(final FileSystem fs, final Configuration conf, final Path edits, final HRegion region) throws IOException { int count = 0; // Read all cells from recover edits List<Cell> walCells = new ArrayList<>(); try (WAL.Reader reader = WALFactory.createReader(fs, edits, conf)) { WAL.Entry entry; while ((entry = reader.next()) != null) { WALKey key = entry.getKey(); WALEdit val = entry.getEdit(); count++; // Check this edit is for this region. if (!Bytes.equals(key.getEncodedRegionName(), region.getRegionInfo().getEncodedNameAsBytes())) { continue; } Cell previous = null; for (Cell cell : val.getCells()) { if (WALEdit.isMetaEditFamily(cell)) { continue; } if (previous != null && CellComparatorImpl.COMPARATOR.compareRows(previous, cell) == 0) { continue; } previous = cell; walCells.add(cell); } } } // Read all cells from region List<Cell> regionCells = new ArrayList<>(); try (RegionScanner scanner = region.getScanner(new Scan())) { List<Cell> tmpCells; do { tmpCells = new ArrayList<>(); scanner.nextRaw(tmpCells); regionCells.addAll(tmpCells); } while (!tmpCells.isEmpty()); } Collections.sort(walCells, CellComparatorImpl.COMPARATOR); int found = 0; for (int i = 0, j = 0; i < walCells.size() && j < regionCells.size(); ) { int compareResult = PrivateCellUtil .compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, walCells.get(i), regionCells.get(j)); if (compareResult == 0) { i++; j++; found++; } else if (compareResult > 0) { j++; } else { i++; } } assertEquals("Only found " + found + " cells in region, but there are " + walCells.size() + " cells in recover edits", found, walCells.size()); return count; }
Example 14
Source File: TestHRegionReplayEvents.java From hbase with Apache License 2.0 | 4 votes |
WAL.Reader createWALReaderForPrimary() throws FileNotFoundException, IOException { return WALFactory.createReader(TEST_UTIL.getTestFileSystem(), AbstractFSWALProvider.getCurrentFileName(walPrimary), TEST_UTIL.getConfiguration()); }