Java Code Examples for org.apache.hadoop.hbase.wal.WAL

The following examples show how to use org.apache.hadoop.hbase.wal.WAL. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1}, null);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example 2
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1}, null);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example 3
Source Project: hbase   Source File: AbstractTestLogRollPeriod.java    License: Apache License 2.0 6 votes vote down vote up
private void checkMinLogRolls(final WAL log, final int minRolls)
    throws Exception {
  final List<Path> paths = new ArrayList<>();
  log.registerWALActionsListener(new WALActionsListener() {
    @Override
    public void postLogRoll(Path oldFile, Path newFile) {
      LOG.debug("postLogRoll: oldFile="+oldFile+" newFile="+newFile);
      paths.add(newFile);
    }
  });

  // Sleep until we should get at least min-LogRoll events
  long wtime = System.currentTimeMillis();
  Thread.sleep((minRolls + 1) * LOG_ROLL_PERIOD);
  // Do some extra sleep in case the machine is slow,
  // and the log-roll is not triggered exactly on LOG_ROLL_PERIOD.
  final int NUM_RETRIES = 1 + 8 * (minRolls - paths.size());
  for (int retry = 0; paths.size() < minRolls && retry < NUM_RETRIES; ++retry) {
    Thread.sleep(LOG_ROLL_PERIOD / 4);
  }
  wtime = System.currentTimeMillis() - wtime;
  LOG.info(String.format("got %d rolls after %dms (%dms each) - expected at least %d rolls",
                         paths.size(), wtime, wtime / paths.size(), minRolls));
  assertFalse(paths.size() < minRolls);
}
 
Example 4
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
                     new LocalRegionServerServices(conf, ServerName.valueOf(
                       InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example 5
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
                     new LocalRegionServerServices(conf, ServerName.valueOf(
                       InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example 6
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example 7
Source Project: hbase   Source File: TestWALConfiguration.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test blocksize change from HBASE-20520 takes on both asycnfs and old wal provider.
 * Hard to verify more than this given the blocksize is passed down to HDFS on create -- not
 * kept local to the streams themselves.
 */
@Test
public void testBlocksizeDefaultsToTwiceHDFSBlockSize() throws IOException {
  TableName tableName = TableName.valueOf("test");
  final WALFactory walFactory = new WALFactory(TEST_UTIL.getConfiguration(), this.walProvider);
  Configuration conf = TEST_UTIL.getConfiguration();
  WALProvider provider = walFactory.getWALProvider();
  // Get a WAL instance from the provider. Check its blocksize.
  WAL wal = provider.getWAL(null);
  if (wal instanceof AbstractFSWAL) {
    long expectedDefaultBlockSize =
        WALUtil.getWALBlockSize(conf, FileSystem.get(conf), TEST_UTIL.getDataTestDir());
    long blocksize = ((AbstractFSWAL)wal).blocksize;
    assertEquals(expectedDefaultBlockSize, blocksize);
    LOG.info("Found blocksize of {} on {}", blocksize, wal);
  } else {
    fail("Unknown provider " + provider);
  }
}
 
Example 8
Source Project: hbase   Source File: MasterRegion.java    License: Apache License 2.0 6 votes vote down vote up
private static HRegion bootstrap(Configuration conf, TableDescriptor td, FileSystem fs,
  Path rootDir, FileSystem walFs, Path walRootDir, WALFactory walFactory,
  MasterRegionWALRoller walRoller, String serverName) throws IOException {
  TableName tn = td.getTableName();
  RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tn).setRegionId(REGION_ID).build();
  Path tmpTableDir = CommonFSUtils.getTableDir(rootDir,
    TableName.valueOf(tn.getNamespaceAsString(), tn.getQualifierAsString() + "-tmp"));
  if (fs.exists(tmpTableDir) && !fs.delete(tmpTableDir, true)) {
    throw new IOException("Can not delete partial created proc region " + tmpTableDir);
  }
  HRegion.createHRegion(conf, regionInfo, fs, tmpTableDir, td).close();
  Path tableDir = CommonFSUtils.getTableDir(rootDir, tn);
  if (!fs.rename(tmpTableDir, tableDir)) {
    throw new IOException("Can not rename " + tmpTableDir + " to " + tableDir);
  }
  WAL wal = createWAL(walFactory, walRoller, serverName, walFs, walRootDir, regionInfo);
  return HRegion.openHRegionFromTableDir(conf, fs, tableDir, regionInfo, td, wal, null, null);
}
 
Example 9
Source Project: hbase   Source File: WALUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * A 'full' WAL transaction involves starting an mvcc transaction followed by an append, an
 * optional sync, and then a call to complete the mvcc transaction. This method does it all. Good
 * for case of adding a single edit or marker to the WAL.
 * <p/>
 * This write is for internal use only. Not for external client consumption.
 * @return WALKeyImpl that was added to the WAL.
 */
private static WALKeyImpl doFullMarkerAppendTransaction(final WAL wal,
  final NavigableMap<byte[], Integer> replicationScope, final RegionInfo hri, final WALEdit edit,
  final MultiVersionConcurrencyControl mvcc,
  final Map<String, byte[]> extendedAttributes, final boolean sync) throws IOException {
  // TODO: Pass in current time to use?
  WALKeyImpl walKey = new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(),
    System.currentTimeMillis(), mvcc, replicationScope, extendedAttributes);
  long trx = MultiVersionConcurrencyControl.NONE;
  try {
    trx = wal.appendMarker(hri, walKey, edit);
    if (sync) {
      wal.sync(trx);
    }
    // Call complete only here because these are markers only. They are not for clients to read.
    mvcc.complete(walKey.getWriteEntry());
  } catch (IOException ioe) {
    if (walKey.getWriteEntry() != null) {
      mvcc.complete(walKey.getWriteEntry());
    }
    throw ioe;
  }
  return walKey;
}
 
Example 10
Source Project: hbase   Source File: TestReplicationEmptyWALRecovery.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Waits until there is only one log(the current writing one) in the replication queue
 * @param numRs number of regionservers
 */
private void waitForLogAdvance(int numRs) throws Exception {
  Waiter.waitFor(CONF1, 10000, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      for (int i = 0; i < numRs; i++) {
        HRegionServer hrs = UTIL1.getHBaseCluster().getRegionServer(i);
        RegionInfo regionInfo =
            UTIL1.getHBaseCluster().getRegions(htable1.getName()).get(0).getRegionInfo();
        WAL wal = hrs.getWAL(regionInfo);
        Path currentFile = ((AbstractFSWAL<?>) wal).getCurrentFileName();
        Replication replicationService = (Replication) UTIL1.getHBaseCluster()
            .getRegionServer(i).getReplicationSourceService();
        for (ReplicationSourceInterface rsi : replicationService.getReplicationManager()
            .getSources()) {
          ReplicationSource source = (ReplicationSource) rsi;
          if (!currentFile.equals(source.getCurrentPath())) {
            return false;
          }
        }
      }
      return true;
    }
  });
}
 
Example 11
Source Project: hbase   Source File: TestHRegionReplayEvents.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test the case where the secondary region replica is not in reads enabled state because it is
 * waiting for a flush or region open marker from primary region. Replaying CANNOT_FLUSH
 * flush marker entry should restore the reads enabled status in the region and allow the reads
 * to continue.
 */
@Test
public void testReplayingFlushRequestRestoresReadsEnabledState() throws IOException {
  disableReads(secondaryRegion);

  // Test case 1: Test that replaying CANNOT_FLUSH request marker assuming this came from
  // triggered flush restores readsEnabled
  primaryRegion.flushcache(true, true, FlushLifeCycleTracker.DUMMY);
  reader = createWALReaderForPrimary();
  while (true) {
    WAL.Entry entry = reader.next();
    if (entry == null) {
      break;
    }
    FlushDescriptor flush = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
    if (flush != null) {
      secondaryRegion.replayWALFlushMarker(flush, entry.getKey().getSequenceId());
    }
  }

  // now reads should be enabled
  secondaryRegion.get(new Get(Bytes.toBytes(0)));
}
 
Example 12
Source Project: hbase   Source File: SerialReplicationTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected final void checkOrder(int expectedEntries) throws IOException {
  try (WAL.Reader reader =
    WALFactory.createReader(UTIL.getTestFileSystem(), logPath, UTIL.getConfiguration())) {
    long seqId = -1L;
    int count = 0;
    for (Entry entry;;) {
      entry = reader.next();
      if (entry == null) {
        break;
      }
      assertTrue(
        "Sequence id go backwards from " + seqId + " to " + entry.getKey().getSequenceId(),
        entry.getKey().getSequenceId() >= seqId);
      seqId = entry.getKey().getSequenceId();
      count++;
    }
    assertEquals(expectedEntries, count);
  }
}
 
Example 13
private long getMaxSeqId(HRegionServer rs, RegionInfo region) throws IOException {
  Path walFile = ((AbstractFSWAL<?>) rs.getWAL(null)).getCurrentFileName();
  long maxSeqId = -1L;
  try (WAL.Reader reader =
    WALFactory.createReader(UTIL.getTestFileSystem(), walFile, UTIL.getConfiguration())) {
    for (;;) {
      WAL.Entry entry = reader.next();
      if (entry == null) {
        break;
      }
      if (Bytes.equals(region.getEncodedNameAsBytes(), entry.getKey().getEncodedRegionName())) {
        maxSeqId = Math.max(maxSeqId, entry.getKey().getSequenceId());
      }
    }
  }
  return maxSeqId;
}
 
Example 14
Source Project: hbase   Source File: TestRegionObserverScannerOpenHook.java    License: Apache License 2.0 6 votes vote down vote up
HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
    byte[]... families) throws IOException {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TableName.valueOf(tableName));
  for (byte[] family : families) {
    tableDescriptor.setColumnFamily(
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family));
  }
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  Path path = new Path(DIR + callingMethod);
  WAL wal = HBaseTestingUtility.createWal(conf, path, info);
  HRegion r = HRegion.createHRegion(info, path, conf, tableDescriptor, wal);
  // this following piece is a hack. currently a coprocessorHost
  // is secretly loaded at OpenRegionHandler. we don't really
  // start a region server here, so just manually create cphost
  // and set it to region.
  RegionCoprocessorHost host = new RegionCoprocessorHost(r, null, conf);
  r.setCoprocessorHost(host);
  return r;
}
 
Example 15
Source Project: hbase   Source File: TestWALObserver.java    License: Apache License 2.0 6 votes vote down vote up
private void addWALEdits(final TableName tableName, final RegionInfo hri, final byte[] rowName,
    final byte[] family, final int count, EnvironmentEdge ee, final WAL wal,
    final NavigableMap<byte[], Integer> scopes, final MultiVersionConcurrencyControl mvcc)
    throws IOException {
  String familyStr = Bytes.toString(family);
  long txid = -1;
  for (int j = 0; j < count; j++) {
    byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
    byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
    WALEdit edit = new WALEdit();
    edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes));
    // uses WALKeyImpl instead of HLogKey on purpose. will only work for tests where we don't care
    // about legacy coprocessors
    txid = wal.appendData(hri,
      new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName, ee.currentTime(), mvcc), edit);
  }
  if (-1 != txid) {
    wal.sync(txid);
  }
}
 
Example 16
Source Project: hbase-indexer   Source File: SepConsumerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReplicateLogEntries_SingleWALEditForMultipleRows() throws IOException {

    byte[] rowKeyA = Bytes.toBytes("A");
    byte[] rowKeyB = Bytes.toBytes("B");
    byte[] data = Bytes.toBytes("data");

    Cell kvA = new KeyValue(rowKeyA, DATA_COLFAM, PAYLOAD_QUALIFIER, data);
    Cell kvB = new KeyValue(rowKeyB, DATA_COLFAM, PAYLOAD_QUALIFIER, data);

    WAL.Entry entry = createHlogEntry(TABLE_NAME, kvA, kvB);

    replicateWALEntry(new WAL.Entry[]{entry});

    SepEvent expectedEventA = SepEvent.create(TABLE_NAME, rowKeyA, Lists.newArrayList(kvA),
            Bytes.toBytes("data"));
    SepEvent expectedEventB = SepEvent.create(TABLE_NAME, rowKeyB, Lists.newArrayList(kvB),
            Bytes.toBytes("data"));

    verify(eventListener).processEvents(Lists.newArrayList(expectedEventA, expectedEventB));
}
 
Example 17
Source Project: phoenix   Source File: SystemCatalogWALEntryFilterIT.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSystemCatalogWALEntryFilter() throws Exception {

  //now create WAL.Entry objects that refer to cells in those view rows in System.Catalog

  Get tenantGet = getGet(catalogTable, TENANT_BYTES, TENANT_VIEW_NAME);
  Get nonTenantGet = getGet(catalogTable, DEFAULT_TENANT_BYTES, NONTENANT_VIEW_NAME);

  WAL.Entry nonTenantEntry = getEntry(systemCatalogTableName, nonTenantGet);
  WAL.Entry tenantEntry = getEntry(systemCatalogTableName, tenantGet);

  //verify that the tenant view WAL.Entry passes the filter and the non-tenant view does not
  SystemCatalogWALEntryFilter filter = new SystemCatalogWALEntryFilter();
  Assert.assertNull(filter.filter(nonTenantEntry));
  WAL.Entry filteredTenantEntry = filter.filter(tenantEntry);
  Assert.assertNotNull("Tenant view was filtered when it shouldn't be!", filteredTenantEntry);
  Assert.assertEquals(tenantEntry.getEdit().size(),
      filter.filter(tenantEntry).getEdit().size());

  //now check that a WAL.Entry with cells from both a tenant and a non-tenant
  //catalog row only allow the tenant cells through
  WALEdit comboEdit = new WALEdit();
  comboEdit.getCells().addAll(nonTenantEntry.getEdit().getCells());
  comboEdit.getCells().addAll(tenantEntry.getEdit().getCells());
  WAL.Entry comboEntry = new WAL.Entry(walKey, comboEdit);

  Assert.assertEquals(tenantEntry.getEdit().size() + nonTenantEntry.getEdit().size()
      , comboEntry.getEdit().size());
  Assert.assertEquals(tenantEntry.getEdit().size(),
      filter.filter(comboEntry).getEdit().size());
}
 
Example 18
private WAL createWAL(final Configuration c, WALFactory walFactory) throws IOException {
  WAL wal = walFactory.getWAL(new byte[]{});

  // Set down maximum recovery so we dfsclient doesn't linger retrying something
  // long gone.
  HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
  return wal;
}
 
Example 19
Source Project: phoenix   Source File: TestPerRegionIndexWriteCache.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
@Before
  public void setUp() throws Exception {
      Path hbaseRootDir = TEST_UTIL.getDataTestDir();
      TEST_UTIL.getConfiguration().set("hbase.rootdir", hbaseRootDir.toString());

      FileSystem newFS = FileSystem.newInstance(TEST_UTIL.getConfiguration());
      HRegionInfo hri = new HRegionInfo(tableName, null, null, false);
      Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
      WALFactory walFactory = new WALFactory(TEST_UTIL.getConfiguration(), null, "TestPerRegionIndexWriteCache");
      WAL wal = walFactory.getWAL(Bytes.toBytes("logs"));
      HTableDescriptor htd = new HTableDescriptor(tableName);
      HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
      htd.addFamily(a);
      
      r1 = new HRegion(basedir, wal, newFS, TEST_UTIL.getConfiguration(), hri, htd, null) {
          @Override
          public int hashCode() {
            return 1;
          }

          @Override
          public String toString() {
            return "testRegion1";
          }
        };
        
      r2 = new HRegion(basedir, wal, newFS, TEST_UTIL.getConfiguration(), hri, htd, null) {
          @Override
          public int hashCode() {
            return 2;
          }

          @Override
          public String toString() {
            return "testRegion1";
          }
        };
  }
 
Example 20
Source Project: hbase   Source File: TestHRegionReplayEvents.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test the case where the secondary region replica is not in reads enabled state because it is
 * waiting for a flush or region open marker from primary region. Replaying region open event
 * entry from primary should restore the reads enabled status in the region and allow the reads
 * to continue.
 */
@Test
public void testReplayingRegionOpenEventRestoresReadsEnabledState() throws IOException {
  // Test case 3: Test that replaying region open event markers restores readsEnabled
  disableReads(secondaryRegion);

  primaryRegion.close();
  primaryRegion = HRegion.openHRegion(rootDir, primaryHri, htd, walPrimary, CONF, rss, null);

  reader = createWALReaderForPrimary();
  while (true) {
    WAL.Entry entry = reader.next();
    if (entry == null) {
      break;
    }

    RegionEventDescriptor regionEventDesc
      = WALEdit.getRegionEventDescriptor(entry.getEdit().getCells().get(0));

    if (regionEventDesc != null) {
      secondaryRegion.replayWALRegionEventMarker(regionEventDesc);
    }
  }

  // now reads should be enabled
  secondaryRegion.get(new Get(Bytes.toBytes(0)));
}
 
Example 21
Source Project: hbase   Source File: MasterRegionWALRoller.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void afterRoll(WAL wal) {
  // move the archived WAL files to the global archive path
  try {
    MasterRegionUtils.moveFilesUnderDir(fs, walArchiveDir, globalWALArchiveDir,
      archivedWALSuffix);
  } catch (IOException e) {
    LOG.warn("Failed to move archived wals from {} to global dir {}", walArchiveDir,
      globalWALArchiveDir, e);
  }
}
 
Example 22
Source Project: hbase   Source File: TestWALActionsListener.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Add a bunch of dummy data and roll the logs every two insert. We
 * should end up with 10 rolled files (plus the roll called in
 * the constructor). Also test adding a listener while it's running.
 */
@Test
public void testActionListener() throws Exception {
  DummyWALActionsListener observer = new DummyWALActionsListener();
  final WALFactory wals = new WALFactory(conf, "testActionListener");
  wals.getWALProvider().addWALActionsListener(observer);
  DummyWALActionsListener laterobserver = new DummyWALActionsListener();
  RegionInfo hri = RegionInfoBuilder.newBuilder(TableName.valueOf(SOME_BYTES))
      .setStartKey(SOME_BYTES).setEndKey(SOME_BYTES).build();
  final WAL wal = wals.getWAL(hri);
  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
  for (int i = 0; i < 20; i++) {
    byte[] b = Bytes.toBytes(i + "");
    KeyValue kv = new KeyValue(b, b, b);
    WALEdit edit = new WALEdit();
    edit.add(kv);
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    scopes.put(b, 0);
    long txid = wal.appendData(hri,
      new WALKeyImpl(hri.getEncodedNameAsBytes(), TableName.valueOf(b), 0, mvcc, scopes), edit);
    wal.sync(txid);
    if (i == 10) {
      wal.registerWALActionsListener(laterobserver);
    }
    if (i % 2 == 0) {
      wal.rollWriter();
    }
  }

  wal.close();

  assertEquals(11, observer.preLogRollCounter);
  assertEquals(11, observer.postLogRollCounter);
  assertEquals(5, laterobserver.preLogRollCounter);
  assertEquals(5, laterobserver.postLogRollCounter);
  assertEquals(1, observer.closedCount);
}
 
Example 23
Source Project: hbase   Source File: CompactingMemStore.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void updateLowestUnflushedSequenceIdInWAL(boolean onlyIfGreater) {
  long minSequenceId = pipeline.getMinSequenceId();
  if(minSequenceId != Long.MAX_VALUE) {
    byte[] encodedRegionName = getRegionServices().getRegionInfo().getEncodedNameAsBytes();
    byte[] familyName = getFamilyNameInBytes();
    WAL WAL = getRegionServices().getWAL();
    if (WAL != null) {
      WAL.updateStore(encodedRegionName, familyName, minSequenceId, onlyIfGreater);
    }
  }
}
 
Example 24
Source Project: hbase   Source File: TestWALReplay.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected WAL createWAL(Configuration c, Path hbaseRootDir, String logName) throws IOException {
  FSHLog wal = new FSHLog(FileSystem.get(c), hbaseRootDir, logName, c);
  wal.init();
  // Set down maximum recovery so we dfsclient doesn't linger retrying something
  // long gone.
  HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
  return wal;
}
 
Example 25
Source Project: hbase   Source File: WALUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Write the marker that a compaction has succeeded and is about to be committed. This provides
 * info to the HMaster to allow it to recover the compaction if this regionserver dies in the
 * middle. It also prevents the compaction from finishing if this regionserver has already lost
 * its lease on the log.
 * <p/>
 * This write is for internal use only. Not for external client consumption.
 * @param mvcc Used by WAL to get sequence Id for the waledit.
 */
public static WALKeyImpl writeCompactionMarker(WAL wal,
  NavigableMap<byte[], Integer> replicationScope, RegionInfo hri, final CompactionDescriptor c,
  MultiVersionConcurrencyControl mvcc) throws IOException {
  WALKeyImpl walKey =
    writeMarker(wal, replicationScope, hri, WALEdit.createCompaction(hri, c), mvcc, null);
  if (LOG.isTraceEnabled()) {
    LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c));
  }
  return walKey;
}
 
Example 26
Source Project: hbase   Source File: WALUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Write a flush marker indicating a start / abort or a complete of a region flush
 * <p/>
 * This write is for internal use only. Not for external client consumption.
 */
public static WALKeyImpl writeFlushMarker(WAL wal, NavigableMap<byte[], Integer> replicationScope,
  RegionInfo hri, final FlushDescriptor f, boolean sync, MultiVersionConcurrencyControl mvcc)
  throws IOException {
  WALKeyImpl walKey = doFullMarkerAppendTransaction(wal, replicationScope, hri,
    WALEdit.createFlushWALEdit(hri, f), mvcc, null, sync);
  if (LOG.isTraceEnabled()) {
    LOG.trace("Appended flush marker " + TextFormat.shortDebugString(f));
  }
  return walKey;
}
 
Example 27
Source Project: hbase   Source File: WALUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Write a region open marker indicating that the region is opened. This write is for internal use
 * only. Not for external client consumption.
 */
public static WALKeyImpl writeRegionEventMarker(WAL wal,
  NavigableMap<byte[], Integer> replicationScope, RegionInfo hri, RegionEventDescriptor r,
  MultiVersionConcurrencyControl mvcc) throws IOException {
  WALKeyImpl walKey =
    writeMarker(wal, replicationScope, hri, WALEdit.createRegionEventWALEdit(hri, r), mvcc, null);
  if (LOG.isTraceEnabled()) {
    LOG.trace("Appended region event marker " + TextFormat.shortDebugString(r));
  }
  return walKey;
}
 
Example 28
Source Project: hbase   Source File: WALUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Write a log marker that a bulk load has succeeded and is about to be committed.
 * This write is for internal use only. Not for external client consumption.
 * @param wal The log to write into.
 * @param replicationScope The replication scope of the families in the HRegion
 * @param hri A description of the region in the table that we are bulk loading into.
 * @param desc A protocol buffers based description of the client's bulk loading request
 * @return walKey with sequenceid filled out for this bulk load marker
 * @throws IOException We will throw an IOException if we can not append to the HLog.
 */
public static WALKeyImpl writeBulkLoadMarkerAndSync(final WAL wal,
    final NavigableMap<byte[], Integer> replicationScope, final RegionInfo hri,
    final WALProtos.BulkLoadDescriptor desc, final MultiVersionConcurrencyControl mvcc)
  throws IOException {
  WALKeyImpl walKey = writeMarker(wal, replicationScope, hri,
    WALEdit.createBulkLoadEvent(hri, desc), mvcc, null);
  if (LOG.isTraceEnabled()) {
    LOG.trace("Appended Bulk Load marker " + TextFormat.shortDebugString(desc));
  }
  return walKey;
}
 
Example 29
Source Project: hbase   Source File: WALUtil.java    License: Apache License 2.0 5 votes vote down vote up
private static WALKeyImpl writeMarker(final WAL wal,
  final NavigableMap<byte[], Integer> replicationScope, final RegionInfo hri, final WALEdit edit,
  final MultiVersionConcurrencyControl mvcc, final Map<String, byte[]> extendedAttributes)
  throws IOException {
  // If sync == true in below, then timeout is not used; safe to pass UNSPECIFIED_TIMEOUT
  return doFullMarkerAppendTransaction(wal, replicationScope, hri, edit, mvcc, extendedAttributes,
    true);
}
 
Example 30
Source Project: hbase   Source File: WALCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Constructor
 * @param log the write ahead log
 * @param conf the configuration
 */
public WALCoprocessorHost(final WAL log, final Configuration conf) {
  // We don't want to require an Abortable passed down through (FS)HLog, so
  // this means that a failure to load of a WAL coprocessor won't abort the
  // server. This isn't ideal, and means that security components that
  // utilize a WALObserver will have to check the observer initialization
  // state manually. However, WALObservers will eventually go away so it
  // should be an acceptable state of affairs.
  super(null);
  this.wal = log;
  // load system default cp's from configuration.
  loadSystemCoprocessors(conf, WAL_COPROCESSOR_CONF_KEY);
}