Java Code Examples for org.apache.hadoop.hbase.wal.WAL#rollWriter()
The following examples show how to use
org.apache.hadoop.hbase.wal.WAL#rollWriter() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AbstractTestLogRolling.java From hbase with Apache License 2.0 | 6 votes |
/** * Tests that logs are deleted */ @Test public void testLogRolling() throws Exception { this.tableName = getName(); // TODO: Why does this write data take for ever? startAndWriteData(); RegionInfo region = server.getRegions(TableName.valueOf(tableName)).get(0).getRegionInfo(); final WAL log = server.getWAL(region); LOG.info("after writing there are " + AbstractFSWALProvider.getNumRolledLogFiles(log) + " log files"); assertLogFileSize(log); // flush all regions for (HRegion r : server.getOnlineRegionsLocalContext()) { r.flush(true); } // Now roll the log log.rollWriter(); int count = AbstractFSWALProvider.getNumRolledLogFiles(log); LOG.info("after flushing all regions and rolling logs there are " + count + " log files"); assertTrue(("actual count: " + count), count <= 2); assertLogFileSize(log); }
Example 2
Source File: TestWALObserver.java From hbase with Apache License 2.0 | 5 votes |
@Test public void testWALObserverRoll() throws Exception { final WAL wal = wals.getWAL(null); final SampleRegionWALCoprocessor cp = getCoprocessor(wal, SampleRegionWALCoprocessor.class); cp.setTestValues(TEST_TABLE, null, null, null, null, null, null, null); assertFalse(cp.isPreWALRollCalled()); assertFalse(cp.isPostWALRollCalled()); wal.rollWriter(true); assertTrue(cp.isPreWALRollCalled()); assertTrue(cp.isPostWALRollCalled()); }
Example 3
Source File: AbstractTestLogRolling.java From hbase with Apache License 2.0 | 5 votes |
/** * Tests that log rolling doesn't hang when no data is written. */ @Test public void testLogRollOnNothingWritten() throws Exception { final Configuration conf = TEST_UTIL.getConfiguration(); final WALFactory wals = new WALFactory(conf, ServerName.valueOf("test.com", 8080, 1).toString()); final WAL newLog = wals.getWAL(null); try { // Now roll the log before we write anything. newLog.rollWriter(true); } finally { wals.close(); } }
Example 4
Source File: TestWALActionsListener.java From hbase with Apache License 2.0 | 5 votes |
/** * Add a bunch of dummy data and roll the logs every two insert. We * should end up with 10 rolled files (plus the roll called in * the constructor). Also test adding a listener while it's running. */ @Test public void testActionListener() throws Exception { DummyWALActionsListener observer = new DummyWALActionsListener(); final WALFactory wals = new WALFactory(conf, "testActionListener"); wals.getWALProvider().addWALActionsListener(observer); DummyWALActionsListener laterobserver = new DummyWALActionsListener(); RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf(SOME_BYTES)) .setStartKey(SOME_BYTES).setEndKey(SOME_BYTES).build(); final WAL wal = wals.getWAL(hri); MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); for (int i = 0; i < 20; i++) { byte[] b = Bytes.toBytes(i + ""); KeyValue kv = new KeyValue(b, b, b); WALEdit edit = new WALEdit(); edit.add(kv); NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); scopes.put(b, 0); long txid = wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), TableName.valueOf(b), 0, mvcc, scopes), edit); wal.sync(txid); if (i == 10) { wal.registerWALActionsListener(laterobserver); } if (i % 2 == 0) { wal.rollWriter(); } } wal.close(); assertEquals(11, observer.preLogRollCounter); assertEquals(11, observer.postLogRollCounter); assertEquals(5, laterobserver.preLogRollCounter); assertEquals(5, laterobserver.postLogRollCounter); assertEquals(1, observer.closedCount); }
Example 5
Source File: TestReplicationSourceManager.java From hbase with Apache License 2.0 | 4 votes |
@Test public void testLogRoll() throws Exception { long baseline = 1000; long time = baseline; MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); KeyValue kv = new KeyValue(r1, f1, r1); WALEdit edit = new WALEdit(); edit.add(kv); WALFactory wals = new WALFactory(utility.getConfiguration(), URLEncoder.encode("regionserver:60020", "UTF8")); ReplicationSourceManager replicationManager = replication.getReplicationManager(); wals.getWALProvider() .addWALActionsListener(new ReplicationSourceWALActionListener(conf, replicationManager)); final WAL wal = wals.getWAL(hri); manager.init(); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf("tableame")) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(f1)).build(); NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); for(byte[] fam : htd.getColumnFamilyNames()) { scopes.put(fam, 0); } // Testing normal log rolling every 20 for(long i = 1; i < 101; i++) { if(i > 1 && i % 20 == 0) { wal.rollWriter(); } LOG.info(Long.toString(i)); final long txid = wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), edit); wal.sync(txid); } // Simulate a rapid insert that's followed // by a report that's still not totally complete (missing last one) LOG.info(baseline + " and " + time); baseline += 101; time = baseline; LOG.info(baseline + " and " + time); for (int i = 0; i < 3; i++) { wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), edit); } wal.sync(); int logNumber = 0; for (Map.Entry<String, NavigableSet<String>> entry : manager.getWALs().get(slaveId) .entrySet()) { logNumber += entry.getValue().size(); } assertEquals(6, logNumber); wal.rollWriter(); ReplicationSourceInterface source = mock(ReplicationSourceInterface.class); when(source.getQueueId()).thenReturn("1"); when(source.isRecovered()).thenReturn(false); when(source.isSyncReplication()).thenReturn(false); manager.logPositionAndCleanOldLogs(source, new WALEntryBatch(0, manager.getSources().get(0).getCurrentPath())); wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), edit); wal.sync(); assertEquals(1, manager.getWALs().size()); // TODO Need a case with only 2 WALs and we only want to delete the first one }
Example 6
Source File: AbstractTestLogRolling.java From hbase with Apache License 2.0 | 4 votes |
/** * Tests that logs are deleted when some region has a compaction * record in WAL and no other records. See HBASE-8597. */ @Test public void testCompactionRecordDoesntBlockRolling() throws Exception { Table table = null; // When the hbase:meta table can be opened, the region servers are running Table t = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME); try { table = createTestTable(getName()); server = TEST_UTIL.getRSForFirstRegionInTable(table.getName()); HRegion region = server.getRegions(table.getName()).get(0); final WAL log = server.getWAL(region.getRegionInfo()); Store s = region.getStore(HConstants.CATALOG_FAMILY); // Put some stuff into table, to make sure we have some files to compact. for (int i = 1; i <= 2; ++i) { doPut(table, i); admin.flush(table.getName()); } doPut(table, 3); // don't flush yet, or compaction might trigger before we roll WAL assertEquals("Should have no WAL after initial writes", 0, AbstractFSWALProvider.getNumRolledLogFiles(log)); assertEquals(2, s.getStorefilesCount()); // Roll the log and compact table, to have compaction record in the 2nd WAL. log.rollWriter(); assertEquals("Should have WAL; one table is not flushed", 1, AbstractFSWALProvider.getNumRolledLogFiles(log)); admin.flush(table.getName()); region.compact(false); // Wait for compaction in case if flush triggered it before us. Assert.assertNotNull(s); for (int waitTime = 3000; s.getStorefilesCount() > 1 && waitTime > 0; waitTime -= 200) { Threads.sleepWithoutInterrupt(200); } assertEquals("Compaction didn't happen", 1, s.getStorefilesCount()); // Write some value to the table so the WAL cannot be deleted until table is flushed. doPut(table, 0); // Now 2nd WAL will have both compaction and put record for table. log.rollWriter(); // 1st WAL deleted, 2nd not deleted yet. assertEquals("Should have WAL; one table is not flushed", 1, AbstractFSWALProvider.getNumRolledLogFiles(log)); // Flush table to make latest WAL obsolete; write another record, and roll again. admin.flush(table.getName()); doPut(table, 1); log.rollWriter(); // Now 2nd WAL is deleted and 3rd is added. assertEquals("Should have 1 WALs at the end", 1, AbstractFSWALProvider.getNumRolledLogFiles(log)); } finally { if (t != null) t.close(); if (table != null) table.close(); } }
Example 7
Source File: TestLogRollAbort.java From hbase with Apache License 2.0 | 4 votes |
/** * Tests the case where a RegionServer enters a GC pause, * comes back online after the master declared it dead and started to split. * Want log rolling after a master split to fail. See HBASE-2312. */ @Test public void testLogRollAfterSplitStart() throws IOException { LOG.info("Verify wal roll after split starts will fail."); String logName = ServerName.valueOf("testLogRollAfterSplitStart", 16010, System.currentTimeMillis()).toString(); Path thisTestsDir = new Path(HBASELOGDIR, AbstractFSWALProvider.getWALDirectoryName(logName)); final WALFactory wals = new WALFactory(conf, logName); try { // put some entries in an WAL TableName tableName = TableName.valueOf(this.getClass().getName()); RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build(); WAL log = wals.getWAL(regionInfo); MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1); int total = 20; for (int i = 0; i < total; i++) { WALEdit kvs = new WALEdit(); kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName())); NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); scopes.put(Bytes.toBytes("column"), 0); log.appendData(regionInfo, new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc, scopes), kvs); } // Send the data to HDFS datanodes and close the HDFS writer log.sync(); ((AbstractFSWAL<?>) log).replaceWriter(((FSHLog)log).getOldPath(), null, null); // code taken from MasterFileSystem.getLogDirs(), which is called from // MasterFileSystem.splitLog() handles RS shutdowns (as observed by the splitting process) // rename the directory so a rogue RS doesn't create more WALs Path rsSplitDir = thisTestsDir.suffix(AbstractFSWALProvider.SPLITTING_EXT); if (!fs.rename(thisTestsDir, rsSplitDir)) { throw new IOException("Failed fs.rename for log split: " + thisTestsDir); } LOG.debug("Renamed region directory: " + rsSplitDir); LOG.debug("Processing the old log files."); WALSplitter.split(HBASELOGDIR, rsSplitDir, OLDLOGDIR, fs, conf, wals); LOG.debug("Trying to roll the WAL."); try { log.rollWriter(); Assert.fail("rollWriter() did not throw any exception."); } catch (IOException ioe) { if (ioe.getCause() instanceof FileNotFoundException) { LOG.info("Got the expected exception: ", ioe.getCause()); } else { Assert.fail("Unexpected exception: " + ioe); } } } finally { wals.close(); if (fs.exists(thisTestsDir)) { fs.delete(thisTestsDir, true); } } }
Example 8
Source File: TestWALPlayer.java From hbase with Apache License 2.0 | 4 votes |
/** * Simple end-to-end test * @throws Exception */ @Test public void testWALPlayer() throws Exception { final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); final byte[] FAMILY = Bytes.toBytes("family"); final byte[] COLUMN1 = Bytes.toBytes("c1"); final byte[] COLUMN2 = Bytes.toBytes("c2"); final byte[] ROW = Bytes.toBytes("row"); Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); Table t2 = TEST_UTIL.createTable(tableName2, FAMILY); // put a row into the first table Put p = new Put(ROW); p.addColumn(FAMILY, COLUMN1, COLUMN1); p.addColumn(FAMILY, COLUMN2, COLUMN2); t1.put(p); // delete one column Delete d = new Delete(ROW); d.addColumns(FAMILY, COLUMN1); t1.delete(d); // replay the WAL, map table 1 to table 2 WAL log = cluster.getRegionServer(0).getWAL(null); log.rollWriter(); String walInputDir = new Path(cluster.getMaster().getMasterFileSystem() .getWALRootDir(), HConstants.HREGION_LOGDIR_NAME).toString(); Configuration configuration= TEST_UTIL.getConfiguration(); WALPlayer player = new WALPlayer(configuration); String optionName="_test_.name"; configuration.set(optionName, "1000"); player.setupTime(configuration, optionName); assertEquals(1000,configuration.getLong(optionName,0)); assertEquals(0, ToolRunner.run(configuration, player, new String[] {walInputDir, tableName1.getNameAsString(), tableName2.getNameAsString() })); // verify the WAL was player into table 2 Get g = new Get(ROW); Result r = t2.get(g); assertEquals(1, r.size()); assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN2)); }