org.apache.hadoop.hbase.wal.WALFactory Java Examples

The following examples show how to use org.apache.hadoop.hbase.wal.WALFactory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestWALMonotonicallyIncreasingSeqId.java    From hbase with Apache License 2.0 6 votes vote down vote up
private HRegion initHRegion(TableDescriptor htd, byte[] startKey, byte[] stopKey, int replicaId)
    throws IOException {
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.set("hbase.wal.provider", walProvider);
  conf.setBoolean("hbase.hregion.mvcc.preassign", false);
  Path tableDir = CommonFSUtils.getTableDir(testDir, htd.getTableName());

  RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(startKey)
      .setEndKey(stopKey).setReplicaId(replicaId).setRegionId(0).build();
  fileSystem = tableDir.getFileSystem(conf);
  final Configuration walConf = new Configuration(conf);
  CommonFSUtils.setRootDir(walConf, tableDir);
  this.walConf = walConf;
  wals = new WALFactory(walConf, "log_" + replicaId);
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  HRegion region = HRegion.createHRegion(info, TEST_UTIL.getDefaultRootDirPath(), conf, htd,
    wals.getWAL(info));
  return region;
}
 
Example #2
Source File: MasterRegion.java    From hbase with Apache License 2.0 6 votes vote down vote up
private static HRegion bootstrap(Configuration conf, TableDescriptor td, FileSystem fs,
  Path rootDir, FileSystem walFs, Path walRootDir, WALFactory walFactory,
  MasterRegionWALRoller walRoller, String serverName) throws IOException {
  TableName tn = td.getTableName();
  RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tn).setRegionId(REGION_ID).build();
  Path tmpTableDir = CommonFSUtils.getTableDir(rootDir,
    TableName.valueOf(tn.getNamespaceAsString(), tn.getQualifierAsString() + "-tmp"));
  if (fs.exists(tmpTableDir) && !fs.delete(tmpTableDir, true)) {
    throw new IOException("Can not delete partial created proc region " + tmpTableDir);
  }
  HRegion.createHRegion(conf, regionInfo, fs, tmpTableDir, td).close();
  Path tableDir = CommonFSUtils.getTableDir(rootDir, tn);
  if (!fs.rename(tmpTableDir, tableDir)) {
    throw new IOException("Can not rename " + tmpTableDir + " to " + tableDir);
  }
  WAL wal = createWAL(walFactory, walRoller, serverName, walFs, walRootDir, regionInfo);
  return HRegion.openHRegionFromTableDir(conf, fs, tableDir, regionInfo, td, wal, null, null);
}
 
Example #3
Source File: MasterRegion.java    From hbase with Apache License 2.0 6 votes vote down vote up
private static WAL createWAL(WALFactory walFactory, MasterRegionWALRoller walRoller,
  String serverName, FileSystem walFs, Path walRootDir, RegionInfo regionInfo)
  throws IOException {
  String logName = AbstractFSWALProvider.getWALDirectoryName(serverName);
  Path walDir = new Path(walRootDir, logName);
  LOG.debug("WALDir={}", walDir);
  if (walFs.exists(walDir)) {
    throw new HBaseIOException(
      "Already created wal directory at " + walDir + " for local region " + regionInfo);
  }
  if (!walFs.mkdirs(walDir)) {
    throw new IOException(
      "Can not create wal directory " + walDir + " for local region " + regionInfo);
  }
  WAL wal = walFactory.getWAL(regionInfo);
  walRoller.addWAL(wal);
  return wal;
}
 
Example #4
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, tableName + ".hlog");
  HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
  WAL hLog = walFactory.getWAL(info);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example #5
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
                     new LocalRegionServerServices(conf, ServerName.valueOf(
                       InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example #6
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
                     new LocalRegionServerServices(conf, ServerName.valueOf(
                       InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example #7
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example #8
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1}, null);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example #9
Source File: TestSyncReplicationActive.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void verifyNoClusterIdInRemoteLog(HBaseTestingUtility utility, Path remoteDir,
    String peerId) throws Exception {
  FileSystem fs2 = utility.getTestFileSystem();
  FileStatus[] files = fs2.listStatus(new Path(remoteDir, peerId));
  Assert.assertTrue(files.length > 0);
  for (FileStatus file : files) {
    try (
      Reader reader = WALFactory.createReader(fs2, file.getPath(), utility.getConfiguration())) {
      Entry entry = reader.next();
      Assert.assertTrue(entry != null);
      while (entry != null) {
        Assert.assertEquals(entry.getKey().getClusterIds().size(), 0);
        entry = reader.next();
      }
    }
  }
}
 
Example #10
Source File: SerialReplicationTestBase.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected final void waitUntilReplicationDone(int expectedEntries) throws Exception {
  UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {

    @Override
    public boolean evaluate() throws Exception {
      try (WAL.Reader reader = WALFactory.createReader(FS, logPath, UTIL.getConfiguration())) {
        int count = 0;
        while (reader.next() != null) {
          count++;
        }
        return count >= expectedEntries;
      } catch (IOException e) {
        return false;
      }
    }

    @Override
    public String explainFailure() throws Exception {
      return "Not enough entries replicated";
    }
  });
}
 
Example #11
Source File: SerialReplicationTestBase.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected final void checkOrder(int expectedEntries) throws IOException {
  try (WAL.Reader reader =
    WALFactory.createReader(UTIL.getTestFileSystem(), logPath, UTIL.getConfiguration())) {
    long seqId = -1L;
    int count = 0;
    for (Entry entry;;) {
      entry = reader.next();
      if (entry == null) {
        break;
      }
      assertTrue(
        "Sequence id go backwards from " + seqId + " to " + entry.getKey().getSequenceId(),
        entry.getKey().getSequenceId() >= seqId);
      seqId = entry.getKey().getSequenceId();
      count++;
    }
    assertEquals(expectedEntries, count);
  }
}
 
Example #12
Source File: TestSequenceIdMonotonicallyIncreasing.java    From hbase with Apache License 2.0 6 votes vote down vote up
private long getMaxSeqId(HRegionServer rs, RegionInfo region) throws IOException {
  Path walFile = ((AbstractFSWAL<?>) rs.getWAL(null)).getCurrentFileName();
  long maxSeqId = -1L;
  try (WAL.Reader reader =
    WALFactory.createReader(UTIL.getTestFileSystem(), walFile, UTIL.getConfiguration())) {
    for (;;) {
      WAL.Entry entry = reader.next();
      if (entry == null) {
        break;
      }
      if (Bytes.equals(region.getEncodedNameAsBytes(), entry.getKey().getEncodedRegionName())) {
        maxSeqId = Math.max(maxSeqId, entry.getKey().getSequenceId());
      }
    }
  }
  return maxSeqId;
}
 
Example #13
Source File: TestWALObserver.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
  // this.cluster = TEST_UTIL.getDFSCluster();
  this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
  this.hbaseRootDir = CommonFSUtils.getRootDir(conf);
  this.hbaseWALRootDir = CommonFSUtils.getWALRootDir(conf);
  this.oldLogDir = new Path(this.hbaseWALRootDir,
      HConstants.HREGION_OLDLOGDIR_NAME);
  String serverName = ServerName.valueOf(currentTest.getMethodName(), 16010,
      System.currentTimeMillis()).toString();
  this.logDir = new Path(this.hbaseWALRootDir,
    AbstractFSWALProvider.getWALDirectoryName(serverName));

  if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
    TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
  }
  if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseWALRootDir)) {
    TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseWALRootDir, true);
  }
  this.wals = new WALFactory(conf, serverName);
}
 
Example #14
Source File: TestHStore.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void initHRegion(String methodName, Configuration conf, TableDescriptorBuilder builder,
    ColumnFamilyDescriptor hcd, MyStoreHook hook, boolean switchToPread) throws IOException {
  TableDescriptor htd = builder.setColumnFamily(hcd).build();
  Path basedir = new Path(DIR + methodName);
  Path tableDir = CommonFSUtils.getTableDir(basedir, htd.getTableName());
  final Path logdir = new Path(basedir, AbstractFSWALProvider.getWALDirectoryName(methodName));

  FileSystem fs = FileSystem.get(conf);

  fs.delete(logdir, true);
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false,
    MemStoreLABImpl.CHUNK_SIZE_DEFAULT, 1, 0, null);
  RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
  Configuration walConf = new Configuration(conf);
  CommonFSUtils.setRootDir(walConf, basedir);
  WALFactory wals = new WALFactory(walConf, methodName);
  region = new HRegion(new HRegionFileSystem(conf, fs, tableDir, info), wals.getWAL(info), conf,
      htd, null);
  region.regionServicesForStores = Mockito.spy(region.regionServicesForStores);
  ThreadPoolExecutor pool = (ThreadPoolExecutor) Executors.newFixedThreadPool(1);
  Mockito.when(region.regionServicesForStores.getInMemoryCompactionPool()).thenReturn(pool);
}
 
Example #15
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1}, null);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example #16
Source File: TestCompactionArchiveIOException.java    From hbase with Apache License 2.0 6 votes vote down vote up
private HRegion initHRegion(TableDescriptor htd, RegionInfo info) throws IOException {
  Configuration conf = testUtil.getConfiguration();
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  Path tableDir = CommonFSUtils.getTableDir(testDir, htd.getTableName());
  Path regionDir = new Path(tableDir, info.getEncodedName());
  Path storeDir = new Path(regionDir, htd.getColumnFamilies()[0].getNameAsString());

  FileSystem errFS = spy(testUtil.getTestFileSystem());
  // Prior to HBASE-16964, when an exception is thrown archiving any compacted file,
  // none of the other files are cleared from the compactedfiles list.
  // Simulate this condition with a dummy file
  doThrow(new IOException("Error for test")).when(errFS)
      .rename(eq(new Path(storeDir, ERROR_FILE)), any());

  HRegionFileSystem fs = new HRegionFileSystem(conf, errFS, tableDir, info);
  final Configuration walConf = new Configuration(conf);
  CommonFSUtils.setRootDir(walConf, tableDir);
  final WALFactory wals = new WALFactory(walConf, "log_" + info.getEncodedName());
  HRegion region = new HRegion(fs, wals.getWAL(info), conf, htd, null);

  region.initialize();

  return region;
}
 
Example #17
Source File: TestCompactionArchiveConcurrentClose.java    From hbase with Apache License 2.0 6 votes vote down vote up
private HRegion initHRegion(TableDescriptor htd, RegionInfo info) throws IOException {
  Configuration conf = testUtil.getConfiguration();
  Path tableDir = CommonFSUtils.getTableDir(testDir, htd.getTableName());

  HRegionFileSystem fs =
      new WaitingHRegionFileSystem(conf, tableDir.getFileSystem(conf), tableDir, info);
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  final Configuration walConf = new Configuration(conf);
  CommonFSUtils.setRootDir(walConf, tableDir);
  final WALFactory wals = new WALFactory(walConf, "log_" + info.getEncodedName());
  HRegion region = new HRegion(fs, wals.getWAL(info), conf, htd, null);

  region.initialize();

  return region;
}
 
Example #18
Source File: AbstractTestWALReplay.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
  this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
  this.hbaseRootDir = CommonFSUtils.getRootDir(this.conf);
  this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
  String serverName =
    ServerName.valueOf(currentTest.getMethodName() + "-manual", 16010, System.currentTimeMillis())
        .toString();
  this.logName = AbstractFSWALProvider.getWALDirectoryName(serverName);
  this.logDir = new Path(this.hbaseRootDir, logName);
  if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
    TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
  }
  this.wals = new WALFactory(conf, currentTest.getMethodName());
}
 
Example #19
Source File: TestDurability.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Test when returnResults set to false in increment it should not return the result instead it
 * resturn null.
 */
@Test
public void testIncrementWithReturnResultsSetToFalse() throws Exception {
  byte[] row1 = Bytes.toBytes("row1");
  byte[] col1 = Bytes.toBytes("col1");

  // Setting up region
  WALFactory wals = new WALFactory(CONF,
      ServerName
          .valueOf("testIncrementWithReturnResultsSetToFalse", 16010, System.currentTimeMillis())
          .toString());
  HRegion region = createHRegion(wals, Durability.USE_DEFAULT);

  Increment inc1 = new Increment(row1);
  inc1.setReturnResults(false);
  inc1.addColumn(FAMILY, col1, 1);
  Result res = region.increment(inc1);
  assertTrue(res.isEmpty());
}
 
Example #20
Source File: TestWALConfiguration.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Test blocksize change from HBASE-20520 takes on both asycnfs and old wal provider.
 * Hard to verify more than this given the blocksize is passed down to HDFS on create -- not
 * kept local to the streams themselves.
 */
@Test
public void testBlocksizeDefaultsToTwiceHDFSBlockSize() throws IOException {
  TableName tableName = TableName.valueOf("test");
  final WALFactory walFactory = new WALFactory(TEST_UTIL.getConfiguration(), this.walProvider);
  Configuration conf = TEST_UTIL.getConfiguration();
  WALProvider provider = walFactory.getWALProvider();
  // Get a WAL instance from the provider. Check its blocksize.
  WAL wal = provider.getWAL(null);
  if (wal instanceof AbstractFSWAL) {
    long expectedDefaultBlockSize =
        WALUtil.getWALBlockSize(conf, FileSystem.get(conf), TEST_UTIL.getDataTestDir());
    long blocksize = ((AbstractFSWAL)wal).blocksize;
    assertEquals(expectedDefaultBlockSize, blocksize);
    LOG.info("Found blocksize of {} on {}", blocksize, wal);
  } else {
    fail("Unknown provider " + provider);
  }
}
 
Example #21
Source File: TestLogRolling.java    From hbase with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  // TODO: testLogRollOnDatanodeDeath fails if short circuit reads are on under the hadoop2
  // profile. See HBASE-9337 for related issues.
  System.setProperty("hbase.tests.use.shortcircuit.reads", "false");

  /**** configuration for testLogRollOnDatanodeDeath ****/
  // lower the namenode & datanode heartbeat so the namenode
  // quickly detects datanode failures
  Configuration conf= TEST_UTIL.getConfiguration();
  conf.setInt("dfs.namenode.heartbeat.recheck-interval", 5000);
  conf.setInt("dfs.heartbeat.interval", 1);
  // the namenode might still try to choose the recently-dead datanode
  // for a pipeline, so try to a new pipeline multiple times
  conf.setInt("dfs.client.block.write.retries", 30);
  conf.setInt("hbase.regionserver.hlog.tolerable.lowreplication", 2);
  conf.setInt("hbase.regionserver.hlog.lowreplication.rolllimit", 3);
  conf.set(WALFactory.WAL_PROVIDER, "filesystem");
  AbstractTestLogRolling.setUpBeforeClass();

  // For slow sync threshold test: roll after 5 slow syncs in 10 seconds
  TEST_UTIL.getConfiguration().setInt(FSHLog.SLOW_SYNC_ROLL_THRESHOLD, 5);
  TEST_UTIL.getConfiguration().setInt(FSHLog.SLOW_SYNC_ROLL_INTERVAL_MS, 10 * 1000);
  // For slow sync threshold test: roll once after a sync above this threshold
  TEST_UTIL.getConfiguration().setInt(FSHLog.ROLL_ON_SYNC_TIME_MS, 5000);
}
 
Example #22
Source File: TestRegionServerCrashDisableWAL.java    From hbase with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {
  UTIL.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
  UTIL.getConfiguration().setBoolean(WALFactory.WAL_ENABLED, false);
  UTIL.startMiniCluster(2);
  UTIL.createTable(TABLE_NAME, CF);
  UTIL.waitTableAvailable(TABLE_NAME);
  HRegionServer rs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
  if (!rs.getRegions(TableName.META_TABLE_NAME).isEmpty()) {
    HRegionServer rs1 = UTIL.getOtherRegionServer(rs);
    UTIL.moveRegionAndWait(
      UTIL.getMiniHBaseCluster().getRegions(TABLE_NAME).get(0).getRegionInfo(),
      rs1.getServerName());
  }
  UTIL.getAdmin().balancerSwitch(false, true);
}
 
Example #23
Source File: TestCompactionInDeadRegionServer.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  UTIL.getConfiguration().setClass(WALFactory.WAL_PROVIDER, walProvider, WALProvider.class);
  UTIL.getConfiguration().setInt(HConstants.ZK_SESSION_TIMEOUT, 2000);
  UTIL.getConfiguration().setClass(HConstants.REGION_SERVER_IMPL, IgnoreYouAreDeadRS.class,
    HRegionServer.class);
  UTIL.startMiniCluster(2);
  Table table = UTIL.createTable(TABLE_NAME, CF);
  for (int i = 0; i < 10; i++) {
    table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
  }
  UTIL.getAdmin().flush(TABLE_NAME);
  for (int i = 10; i < 20; i++) {
    table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i)));
  }
  UTIL.getAdmin().flush(TABLE_NAME);
}
 
Example #24
Source File: TestHMobStore.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void init(String methodName, Configuration conf, ColumnFamilyDescriptor cfd,
    boolean testStore) throws IOException {
  TableDescriptor td =
      TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setColumnFamily(cfd).build();

  //Setting up tje Region and Store
  Path basedir = new Path(DIR + methodName);
  Path tableDir = CommonFSUtils.getTableDir(basedir, td.getTableName());
  String logName = "logs";
  Path logdir = new Path(basedir, logName);
  FileSystem fs = FileSystem.get(conf);
  fs.delete(logdir, true);

  RegionInfo info = RegionInfoBuilder.newBuilder(td.getTableName()).build();
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  final Configuration walConf = new Configuration(conf);
  CommonFSUtils.setRootDir(walConf, basedir);
  final WALFactory wals = new WALFactory(walConf, methodName);
  region = new HRegion(tableDir, wals.getWAL(info), fs, conf, info, td, null);
  region.setMobFileCache(new MobFileCache(conf));
  store = new HMobStore(region, cfd, conf, false);
  if (testStore) {
    init(conf, cfd);
  }
}
 
Example #25
Source File: TestStoreFileRefresherChore.java    From hbase with Apache License 2.0 6 votes vote down vote up
private HRegion initHRegion(TableDescriptor htd, byte[] startKey, byte[] stopKey, int replicaId)
    throws IOException {
  Configuration conf = TEST_UTIL.getConfiguration();
  Path tableDir = CommonFSUtils.getTableDir(testDir, htd.getTableName());

  RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(startKey)
      .setEndKey(stopKey).setRegionId(0L).setReplicaId(replicaId).build();
  HRegionFileSystem fs =
      new FailingHRegionFileSystem(conf, tableDir.getFileSystem(conf), tableDir, info);
  final Configuration walConf = new Configuration(conf);
  CommonFSUtils.setRootDir(walConf, tableDir);
  final WALFactory wals = new WALFactory(walConf, "log_" + replicaId);
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  HRegion region =
      new HRegion(fs, wals.getWAL(info),
          conf, htd, null);

  region.initialize();

  return region;
}
 
Example #26
Source File: TestAsyncLogRolling.java    From hbase with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  Configuration conf = TestAsyncLogRolling.TEST_UTIL.getConfiguration();
  conf.setInt(FanOutOneBlockAsyncDFSOutputHelper.ASYNC_DFS_OUTPUT_CREATE_MAX_RETRIES, 100);
  conf.set(WALFactory.WAL_PROVIDER, "asyncfs");
  AbstractTestLogRolling.setUpBeforeClass();
}
 
Example #27
Source File: WALEntryStream.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void openReader(Path path) throws IOException {
  try {
    // Detect if this is a new file, if so get a new reader else
    // reset the current reader so that we see the new data
    if (reader == null || !getCurrentPath().equals(path)) {
      closeReader();
      reader = WALFactory.createReader(fs, path, conf);
      seek();
      setCurrentPath(path);
    } else {
      resetReader();
    }
  } catch (FileNotFoundException fnfe) {
    handleFileNotFound(path, fnfe);
  }  catch (RemoteException re) {
    IOException ioe = re.unwrapRemoteException(FileNotFoundException.class);
    if (!(ioe instanceof FileNotFoundException)) throw ioe;
    handleFileNotFound(path, (FileNotFoundException)ioe);
  } catch (LeaseNotRecoveredException lnre) {
    // HBASE-15019 the WAL was not closed due to some hiccup.
    LOG.warn("Try to recover the WAL lease " + currentPath, lnre);
    recoverLease(conf, currentPath);
    reader = null;
  } catch (NullPointerException npe) {
    // Workaround for race condition in HDFS-4380
    // which throws a NPE if we open a file before any data node has the most recent block
    // Just sleep and retry. Will require re-reading compressed WALs for compressionContext.
    LOG.warn("Got NPE opening reader, will retry.");
    reader = null;
  }
}
 
Example #28
Source File: TestDefaultMemStore.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testShouldFlushMeta() throws Exception {
  // write an edit in the META and ensure the shouldFlush (that the periodic memstore
  // flusher invokes) returns true after SYSTEM_CACHE_FLUSH_INTERVAL (even though
  // the MEMSTORE_PERIODIC_FLUSH_INTERVAL is set to a higher value)
  Configuration conf = new Configuration();
  conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, HRegion.SYSTEM_CACHE_FLUSH_INTERVAL * 10);
  HBaseTestingUtility hbaseUtility = new HBaseTestingUtility(conf);
  Path testDir = hbaseUtility.getDataTestDir();
  EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
  EnvironmentEdgeManager.injectEdge(edge);
  edge.setCurrentTimeMillis(1234);
  WALFactory wFactory = new WALFactory(conf, "1234");
  TableDescriptors tds = new FSTableDescriptors(conf);
  FSTableDescriptors.tryUpdateMetaTableDescriptor(conf);
  HRegion meta = HRegion.createHRegion(RegionInfoBuilder.FIRST_META_REGIONINFO, testDir,
      conf, tds.get(TableName.META_TABLE_NAME),
      wFactory.getWAL(RegionInfoBuilder.FIRST_META_REGIONINFO));
  // parameterized tests add [#] suffix get rid of [ and ].
  TableDescriptor desc = TableDescriptorBuilder
      .newBuilder(TableName.valueOf(name.getMethodName().replaceAll("[\\[\\]]", "_")))
      .setColumnFamily(ColumnFamilyDescriptorBuilder.of("foo")).build();
  RegionInfo hri = RegionInfoBuilder.newBuilder(desc.getTableName())
      .setStartKey(Bytes.toBytes("row_0200")).setEndKey(Bytes.toBytes("row_0300")).build();
  HRegion r = HRegion.createHRegion(hri, testDir, conf, desc, wFactory.getWAL(hri));
  addRegionToMETA(meta, r);
  edge.setCurrentTimeMillis(1234 + 100);
  StringBuilder sb = new StringBuilder();
  assertTrue(meta.shouldFlush(sb) == false);
  edge.setCurrentTimeMillis(edge.currentTime() + HRegion.SYSTEM_CACHE_FLUSH_INTERVAL + 1);
  assertTrue(meta.shouldFlush(sb) == true);
}
 
Example #29
Source File: ReplaySyncReplicationWALCallable.java    From hbase with Apache License 2.0 5 votes vote down vote up
private Reader getReader(String wal) throws IOException {
  Path path = new Path(rs.getWALRootDir(), wal);
  long length = rs.getWALFileSystem().getFileStatus(path).getLen();
  try {
    RecoverLeaseFSUtils.recoverFileLease(fs, path, conf);
    return WALFactory.createReader(rs.getWALFileSystem(), path, rs.getConfiguration());
  } catch (EOFException e) {
    if (length <= 0) {
      LOG.warn("File is empty. Could not open {} for reading because {}", path, e);
      return null;
    }
    throw e;
  }
}
 
Example #30
Source File: MasterRegion.java    From hbase with Apache License 2.0 5 votes vote down vote up
private MasterRegion(HRegion region, WALFactory walFactory,
  MasterRegionFlusherAndCompactor flusherAndCompactor, MasterRegionWALRoller walRoller) {
  this.region = region;
  this.walFactory = walFactory;
  this.flusherAndCompactor = flusherAndCompactor;
  this.walRoller = walRoller;
}