Java Code Examples for org.apache.hadoop.hbase.wal.WAL#sync()
The following examples show how to use
org.apache.hadoop.hbase.wal.WAL#sync() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: WALUtil.java From hbase with Apache License 2.0 | 6 votes |
/** * A 'full' WAL transaction involves starting an mvcc transaction followed by an append, an * optional sync, and then a call to complete the mvcc transaction. This method does it all. Good * for case of adding a single edit or marker to the WAL. * <p/> * This write is for internal use only. Not for external client consumption. * @return WALKeyImpl that was added to the WAL. */ private static WALKeyImpl doFullMarkerAppendTransaction(final WAL wal, final NavigableMap<byte[], Integer> replicationScope, final RegionInfo hri, final WALEdit edit, final MultiVersionConcurrencyControl mvcc, final Map<String, byte[]> extendedAttributes, final boolean sync) throws IOException { // TODO: Pass in current time to use? WALKeyImpl walKey = new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(), System.currentTimeMillis(), mvcc, replicationScope, extendedAttributes); long trx = MultiVersionConcurrencyControl.NONE; try { trx = wal.appendMarker(hri, walKey, edit); if (sync) { wal.sync(trx); } // Call complete only here because these are markers only. They are not for clients to read. mvcc.complete(walKey.getWriteEntry()); } catch (IOException ioe) { if (walKey.getWriteEntry() != null) { mvcc.complete(walKey.getWriteEntry()); } throw ioe; } return walKey; }
Example 2
Source File: TestWALObserver.java From hbase with Apache License 2.0 | 6 votes |
private void addWALEdits(final TableName tableName, final RegionInfo hri, final byte[] rowName, final byte[] family, final int count, EnvironmentEdge ee, final WAL wal, final NavigableMap<byte[], Integer> scopes, final MultiVersionConcurrencyControl mvcc) throws IOException { String familyStr = Bytes.toString(family); long txid = -1; for (int j = 0; j < count; j++) { byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j)); byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j)); WALEdit edit = new WALEdit(); edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes)); // uses WALKeyImpl instead of HLogKey on purpose. will only work for tests where we don't care // about legacy coprocessors txid = wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, ee.currentTime(), mvcc), edit); } if (-1 != txid) { wal.sync(txid); } }
Example 3
Source File: TestWALObserver.java From hbase with Apache License 2.0 | 5 votes |
/** * Coprocessors shouldn't get notice of empty waledits. */ @Test public void testEmptyWALEditAreNotSeen() throws Exception { RegionInfo hri = createBasicHRegionInfo(Bytes.toString(TEST_TABLE)); TableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE)); MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); for(byte[] fam : htd.getColumnFamilyNames()) { scopes.put(fam, 0); } WAL log = wals.getWAL(null); try { SampleRegionWALCoprocessor cp = getCoprocessor(log, SampleRegionWALCoprocessor.class); cp.setTestValues(TEST_TABLE, null, null, null, null, null, null, null); assertFalse(cp.isPreWALWriteCalled()); assertFalse(cp.isPostWALWriteCalled()); final long now = EnvironmentEdgeManager.currentTime(); long txid = log.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(), now, mvcc, scopes), new WALEdit()); log.sync(txid); assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPreWALWriteCalled()); assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPostWALWriteCalled()); } finally { log.close(); } }
Example 4
Source File: ProtobufLogTestHelper.java From hbase with Apache License 2.0 | 5 votes |
public static void doWrite(WAL wal, RegionInfo hri, TableName tableName, int columnCount, int recordCount, byte[] row, long timestamp, MultiVersionConcurrencyControl mvcc) throws IOException { for (int i = 0; i < recordCount; i++) { WAL.Entry entry = generateEdit(i, hri, tableName, row, columnCount, timestamp, mvcc); wal.appendData(hri, entry.getKey(), entry.getEdit()); } wal.sync(); }
Example 5
Source File: AbstractTestFSWAL.java From hbase with Apache License 2.0 | 5 votes |
protected void addEdits(WAL log, RegionInfo hri, TableDescriptor htd, int times, MultiVersionConcurrencyControl mvcc, NavigableMap<byte[], Integer> scopes, String cf) throws IOException { final byte[] row = Bytes.toBytes(cf); for (int i = 0; i < times; i++) { long timestamp = System.currentTimeMillis(); WALEdit cols = new WALEdit(); cols.add(new KeyValue(row, row, row, timestamp, row)); WALKeyImpl key = new WALKeyImpl(hri.getEncodedNameAsBytes(), htd.getTableName(), SequenceId.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE, HConstants.NO_NONCE, mvcc, scopes); log.appendData(hri, key, cols); } log.sync(); }
Example 6
Source File: AbstractTestWALReplay.java From hbase with Apache License 2.0 | 5 votes |
private void addWALEdits(final TableName tableName, final RegionInfo hri, final byte[] rowName, final byte[] family, final int count, EnvironmentEdge ee, final WAL wal, final MultiVersionConcurrencyControl mvcc, NavigableMap<byte[], Integer> scopes) throws IOException { for (int j = 0; j < count; j++) { wal.appendData(hri, createWALKey(tableName, hri, mvcc, scopes), createWALEdit(rowName, family, ee, j)); } wal.sync(); }
Example 7
Source File: TestWALActionsListener.java From hbase with Apache License 2.0 | 5 votes |
/** * Add a bunch of dummy data and roll the logs every two insert. We * should end up with 10 rolled files (plus the roll called in * the constructor). Also test adding a listener while it's running. */ @Test public void testActionListener() throws Exception { DummyWALActionsListener observer = new DummyWALActionsListener(); final WALFactory wals = new WALFactory(conf, "testActionListener"); wals.getWALProvider().addWALActionsListener(observer); DummyWALActionsListener laterobserver = new DummyWALActionsListener(); RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf(SOME_BYTES)) .setStartKey(SOME_BYTES).setEndKey(SOME_BYTES).build(); final WAL wal = wals.getWAL(hri); MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); for (int i = 0; i < 20; i++) { byte[] b = Bytes.toBytes(i + ""); KeyValue kv = new KeyValue(b, b, b); WALEdit edit = new WALEdit(); edit.add(kv); NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); scopes.put(b, 0); long txid = wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), TableName.valueOf(b), 0, mvcc, scopes), edit); wal.sync(txid); if (i == 10) { wal.registerWALActionsListener(laterobserver); } if (i % 2 == 0) { wal.rollWriter(); } } wal.close(); assertEquals(11, observer.preLogRollCounter); assertEquals(11, observer.postLogRollCounter); assertEquals(5, laterobserver.preLogRollCounter); assertEquals(5, laterobserver.postLogRollCounter); assertEquals(1, observer.closedCount); }
Example 8
Source File: TestReplicationSourceManager.java From hbase with Apache License 2.0 | 4 votes |
@Test public void testLogRoll() throws Exception { long baseline = 1000; long time = baseline; MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); KeyValue kv = new KeyValue(r1, f1, r1); WALEdit edit = new WALEdit(); edit.add(kv); WALFactory wals = new WALFactory(utility.getConfiguration(), URLEncoder.encode("regionserver:60020", "UTF8")); ReplicationSourceManager replicationManager = replication.getReplicationManager(); wals.getWALProvider() .addWALActionsListener(new ReplicationSourceWALActionListener(conf, replicationManager)); final WAL wal = wals.getWAL(hri); manager.init(); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf("tableame")) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(f1)).build(); NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); for(byte[] fam : htd.getColumnFamilyNames()) { scopes.put(fam, 0); } // Testing normal log rolling every 20 for(long i = 1; i < 101; i++) { if(i > 1 && i % 20 == 0) { wal.rollWriter(); } LOG.info(Long.toString(i)); final long txid = wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), edit); wal.sync(txid); } // Simulate a rapid insert that's followed // by a report that's still not totally complete (missing last one) LOG.info(baseline + " and " + time); baseline += 101; time = baseline; LOG.info(baseline + " and " + time); for (int i = 0; i < 3; i++) { wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), edit); } wal.sync(); int logNumber = 0; for (Map.Entry<String, NavigableSet<String>> entry : manager.getWALs().get(slaveId) .entrySet()) { logNumber += entry.getValue().size(); } assertEquals(6, logNumber); wal.rollWriter(); ReplicationSourceInterface source = mock(ReplicationSourceInterface.class); when(source.getQueueId()).thenReturn("1"); when(source.isRecovered()).thenReturn(false); when(source.isSyncReplication()).thenReturn(false); manager.logPositionAndCleanOldLogs(source, new WALEntryBatch(0, manager.getSources().get(0).getCurrentPath())); wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes), edit); wal.sync(); assertEquals(1, manager.getWALs().size()); // TODO Need a case with only 2 WALs and we only want to delete the first one }
Example 9
Source File: TestDurability.java From hbase with Apache License 2.0 | 4 votes |
@Test public void testDurability() throws Exception { WALFactory wals = new WALFactory(CONF, ServerName.valueOf("TestDurability", 16010, System.currentTimeMillis()).toString()); HRegion region = createHRegion(wals, Durability.USE_DEFAULT); WAL wal = region.getWAL(); HRegion deferredRegion = createHRegion(region.getTableDescriptor(), region.getRegionInfo(), "deferredRegion", wal, Durability.ASYNC_WAL); region.put(newPut(null)); verifyWALCount(wals, wal, 1); // a put through the deferred table does not write to the wal immediately, // but maybe has been successfully sync-ed by the underlying AsyncWriter + // AsyncFlusher thread deferredRegion.put(newPut(null)); // but will after we sync the wal wal.sync(); verifyWALCount(wals, wal, 2); // a put through a deferred table will be sync with the put sync'ed put deferredRegion.put(newPut(null)); wal.sync(); verifyWALCount(wals, wal, 3); region.put(newPut(null)); verifyWALCount(wals, wal, 4); // a put through a deferred table will be sync with the put sync'ed put deferredRegion.put(newPut(Durability.USE_DEFAULT)); wal.sync(); verifyWALCount(wals, wal, 5); region.put(newPut(Durability.USE_DEFAULT)); verifyWALCount(wals, wal, 6); // SKIP_WAL never writes to the wal region.put(newPut(Durability.SKIP_WAL)); deferredRegion.put(newPut(Durability.SKIP_WAL)); verifyWALCount(wals, wal, 6); wal.sync(); verifyWALCount(wals, wal, 6); // Async overrides sync table default region.put(newPut(Durability.ASYNC_WAL)); deferredRegion.put(newPut(Durability.ASYNC_WAL)); wal.sync(); verifyWALCount(wals, wal, 8); // sync overrides async table default region.put(newPut(Durability.SYNC_WAL)); deferredRegion.put(newPut(Durability.SYNC_WAL)); verifyWALCount(wals, wal, 10); // fsync behaves like sync region.put(newPut(Durability.FSYNC_WAL)); deferredRegion.put(newPut(Durability.FSYNC_WAL)); verifyWALCount(wals, wal, 12); }
Example 10
Source File: TestLogRollAbort.java From hbase with Apache License 2.0 | 4 votes |
/** * Tests the case where a RegionServer enters a GC pause, * comes back online after the master declared it dead and started to split. * Want log rolling after a master split to fail. See HBASE-2312. */ @Test public void testLogRollAfterSplitStart() throws IOException { LOG.info("Verify wal roll after split starts will fail."); String logName = ServerName.valueOf("testLogRollAfterSplitStart", 16010, System.currentTimeMillis()).toString(); Path thisTestsDir = new Path(HBASELOGDIR, AbstractFSWALProvider.getWALDirectoryName(logName)); final WALFactory wals = new WALFactory(conf, logName); try { // put some entries in an WAL TableName tableName = TableName.valueOf(this.getClass().getName()); RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build(); WAL log = wals.getWAL(regionInfo); MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1); int total = 20; for (int i = 0; i < total; i++) { WALEdit kvs = new WALEdit(); kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName())); NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); scopes.put(Bytes.toBytes("column"), 0); log.appendData(regionInfo, new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc, scopes), kvs); } // Send the data to HDFS datanodes and close the HDFS writer log.sync(); ((AbstractFSWAL<?>) log).replaceWriter(((FSHLog)log).getOldPath(), null, null); // code taken from MasterFileSystem.getLogDirs(), which is called from // MasterFileSystem.splitLog() handles RS shutdowns (as observed by the splitting process) // rename the directory so a rogue RS doesn't create more WALs Path rsSplitDir = thisTestsDir.suffix(AbstractFSWALProvider.SPLITTING_EXT); if (!fs.rename(thisTestsDir, rsSplitDir)) { throw new IOException("Failed fs.rename for log split: " + thisTestsDir); } LOG.debug("Renamed region directory: " + rsSplitDir); LOG.debug("Processing the old log files."); WALSplitter.split(HBASELOGDIR, rsSplitDir, OLDLOGDIR, fs, conf, wals); LOG.debug("Trying to roll the WAL."); try { log.rollWriter(); Assert.fail("rollWriter() did not throw any exception."); } catch (IOException ioe) { if (ioe.getCause() instanceof FileNotFoundException) { LOG.info("Got the expected exception: ", ioe.getCause()); } else { Assert.fail("Unexpected exception: " + ioe); } } } finally { wals.close(); if (fs.exists(thisTestsDir)) { fs.delete(thisTestsDir, true); } } }