Java Code Examples for org.apache.hadoop.hbase.wal.WALFactory#getWAL()

The following examples show how to use org.apache.hadoop.hbase.wal.WALFactory#getWAL() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1}, null);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example 2
Source File: TestCompactionArchiveIOException.java    From hbase with Apache License 2.0 6 votes vote down vote up
private HRegion initHRegion(TableDescriptor htd, RegionInfo info) throws IOException {
  Configuration conf = testUtil.getConfiguration();
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  Path tableDir = CommonFSUtils.getTableDir(testDir, htd.getTableName());
  Path regionDir = new Path(tableDir, info.getEncodedName());
  Path storeDir = new Path(regionDir, htd.getColumnFamilies()[0].getNameAsString());

  FileSystem errFS = spy(testUtil.getTestFileSystem());
  // Prior to HBASE-16964, when an exception is thrown archiving any compacted file,
  // none of the other files are cleared from the compactedfiles list.
  // Simulate this condition with a dummy file
  doThrow(new IOException("Error for test")).when(errFS)
      .rename(eq(new Path(storeDir, ERROR_FILE)), any());

  HRegionFileSystem fs = new HRegionFileSystem(conf, errFS, tableDir, info);
  final Configuration walConf = new Configuration(conf);
  CommonFSUtils.setRootDir(walConf, tableDir);
  final WALFactory wals = new WALFactory(walConf, "log_" + info.getEncodedName());
  HRegion region = new HRegion(fs, wals.getWAL(info), conf, htd, null);

  region.initialize();

  return region;
}
 
Example 3
Source File: TestCompactionArchiveConcurrentClose.java    From hbase with Apache License 2.0 6 votes vote down vote up
private HRegion initHRegion(TableDescriptor htd, RegionInfo info) throws IOException {
  Configuration conf = testUtil.getConfiguration();
  Path tableDir = CommonFSUtils.getTableDir(testDir, htd.getTableName());

  HRegionFileSystem fs =
      new WaitingHRegionFileSystem(conf, tableDir.getFileSystem(conf), tableDir, info);
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  final Configuration walConf = new Configuration(conf);
  CommonFSUtils.setRootDir(walConf, tableDir);
  final WALFactory wals = new WALFactory(walConf, "log_" + info.getEncodedName());
  HRegion region = new HRegion(fs, wals.getWAL(info), conf, htd, null);

  region.initialize();

  return region;
}
 
Example 4
Source File: TestHMobStore.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void init(String methodName, Configuration conf, ColumnFamilyDescriptor cfd,
    boolean testStore) throws IOException {
  TableDescriptor td =
      TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setColumnFamily(cfd).build();

  //Setting up tje Region and Store
  Path basedir = new Path(DIR + methodName);
  Path tableDir = CommonFSUtils.getTableDir(basedir, td.getTableName());
  String logName = "logs";
  Path logdir = new Path(basedir, logName);
  FileSystem fs = FileSystem.get(conf);
  fs.delete(logdir, true);

  RegionInfo info = RegionInfoBuilder.newBuilder(td.getTableName()).build();
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  final Configuration walConf = new Configuration(conf);
  CommonFSUtils.setRootDir(walConf, basedir);
  final WALFactory wals = new WALFactory(walConf, methodName);
  region = new HRegion(tableDir, wals.getWAL(info), fs, conf, info, td, null);
  region.setMobFileCache(new MobFileCache(conf));
  store = new HMobStore(region, cfd, conf, false);
  if (testStore) {
    init(conf, cfd);
  }
}
 
Example 5
Source File: TestHStore.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void initHRegion(String methodName, Configuration conf, TableDescriptorBuilder builder,
    ColumnFamilyDescriptor hcd, MyStoreHook hook, boolean switchToPread) throws IOException {
  TableDescriptor htd = builder.setColumnFamily(hcd).build();
  Path basedir = new Path(DIR + methodName);
  Path tableDir = CommonFSUtils.getTableDir(basedir, htd.getTableName());
  final Path logdir = new Path(basedir, AbstractFSWALProvider.getWALDirectoryName(methodName));

  FileSystem fs = FileSystem.get(conf);

  fs.delete(logdir, true);
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false,
    MemStoreLABImpl.CHUNK_SIZE_DEFAULT, 1, 0, null);
  RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
  Configuration walConf = new Configuration(conf);
  CommonFSUtils.setRootDir(walConf, basedir);
  WALFactory wals = new WALFactory(walConf, methodName);
  region = new HRegion(new HRegionFileSystem(conf, fs, tableDir, info), wals.getWAL(info), conf,
      htd, null);
  region.regionServicesForStores = Mockito.spy(region.regionServicesForStores);
  ThreadPoolExecutor pool = (ThreadPoolExecutor) Executors.newFixedThreadPool(1);
  Mockito.when(region.regionServicesForStores.getInMemoryCompactionPool()).thenReturn(pool);
}
 
Example 6
Source File: TestStoreFileRefresherChore.java    From hbase with Apache License 2.0 6 votes vote down vote up
private HRegion initHRegion(TableDescriptor htd, byte[] startKey, byte[] stopKey, int replicaId)
    throws IOException {
  Configuration conf = TEST_UTIL.getConfiguration();
  Path tableDir = CommonFSUtils.getTableDir(testDir, htd.getTableName());

  RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(startKey)
      .setEndKey(stopKey).setRegionId(0L).setReplicaId(replicaId).build();
  HRegionFileSystem fs =
      new FailingHRegionFileSystem(conf, tableDir.getFileSystem(conf), tableDir, info);
  final Configuration walConf = new Configuration(conf);
  CommonFSUtils.setRootDir(walConf, tableDir);
  final WALFactory wals = new WALFactory(walConf, "log_" + replicaId);
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  HRegion region =
      new HRegion(fs, wals.getWAL(info),
          conf, htd, null);

  region.initialize();

  return region;
}
 
Example 7
Source File: MasterRegion.java    From hbase with Apache License 2.0 6 votes vote down vote up
private static WAL createWAL(WALFactory walFactory, MasterRegionWALRoller walRoller,
  String serverName, FileSystem walFs, Path walRootDir, RegionInfo regionInfo)
  throws IOException {
  String logName = AbstractFSWALProvider.getWALDirectoryName(serverName);
  Path walDir = new Path(walRootDir, logName);
  LOG.debug("WALDir={}", walDir);
  if (walFs.exists(walDir)) {
    throw new HBaseIOException(
      "Already created wal directory at " + walDir + " for local region " + regionInfo);
  }
  if (!walFs.mkdirs(walDir)) {
    throw new IOException(
      "Can not create wal directory " + walDir + " for local region " + regionInfo);
  }
  WAL wal = walFactory.getWAL(regionInfo);
  walRoller.addWAL(wal);
  return wal;
}
 
Example 8
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example 9
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
                     new LocalRegionServerServices(conf, ServerName.valueOf(
                       InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example 10
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, tableName + ".hlog");
  HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
  WAL hLog = walFactory.getWAL(info);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example 11
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1}, null);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example 12
Source File: TestPerRegionIndexWriteCache.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
@Before
  public void setUp() throws Exception {
      Path hbaseRootDir = TEST_UTIL.getDataTestDir();
      TEST_UTIL.getConfiguration().set("hbase.rootdir", hbaseRootDir.toString());

      FileSystem newFS = FileSystem.newInstance(TEST_UTIL.getConfiguration());
      HRegionInfo hri = new HRegionInfo(tableName, null, null, false);
      Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
      WALFactory walFactory = new WALFactory(TEST_UTIL.getConfiguration(), null, "TestPerRegionIndexWriteCache");
      WAL wal = walFactory.getWAL(Bytes.toBytes("logs"));
      HTableDescriptor htd = new HTableDescriptor(tableName);
      HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
      htd.addFamily(a);
      
      r1 = new HRegion(basedir, wal, newFS, TEST_UTIL.getConfiguration(), hri, htd, null) {
          @Override
          public int hashCode() {
            return 1;
          }

          @Override
          public String toString() {
            return "testRegion1";
          }
        };
        
      r2 = new HRegion(basedir, wal, newFS, TEST_UTIL.getConfiguration(), hri, htd, null) {
          @Override
          public int hashCode() {
            return 2;
          }

          @Override
          public String toString() {
            return "testRegion1";
          }
        };
  }
 
Example 13
Source File: WALReplayWithIndexWritesAndCompressedWALIT.java    From phoenix with Apache License 2.0 5 votes vote down vote up
private WAL createWAL(final Configuration c, WALFactory walFactory) throws IOException {
  WAL wal = walFactory.getWAL(new byte[]{});

  // Set down maximum recovery so we dfsclient doesn't linger retrying something
  // long gone.
  HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
  return wal;
}
 
Example 14
Source File: TestWALEntryStream.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
  walQueue = new PriorityBlockingQueue<>();
  pathWatcher = new PathWatcher();
  final WALFactory wals = new WALFactory(CONF, tn.getMethodName());
  wals.getWALProvider().addWALActionsListener(pathWatcher);
  log = wals.getWAL(info);
}
 
Example 15
Source File: AbstractTestLogRolling.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that log rolling doesn't hang when no data is written.
 */
@Test
public void testLogRollOnNothingWritten() throws Exception {
  final Configuration conf = TEST_UTIL.getConfiguration();
  final WALFactory wals =
    new WALFactory(conf, ServerName.valueOf("test.com", 8080, 1).toString());
  final WAL newLog = wals.getWAL(null);
  try {
    // Now roll the log before we write anything.
    newLog.rollWriter(true);
  } finally {
    wals.close();
  }
}
 
Example 16
Source File: WALReplayWithIndexWritesAndCompressedWALIT.java    From phoenix with Apache License 2.0 5 votes vote down vote up
private WAL createWAL(final Configuration c, WALFactory walFactory) throws IOException {
  WAL wal = walFactory.getWAL(null);

  // Set down maximum recovery so we dfsclient doesn't linger retrying something
  // long gone.
  HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
  return wal;
}
 
Example 17
Source File: TestReplicationSourceManager.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Test
public void testLogRoll() throws Exception {
  long baseline = 1000;
  long time = baseline;
  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
  KeyValue kv = new KeyValue(r1, f1, r1);
  WALEdit edit = new WALEdit();
  edit.add(kv);

  WALFactory wals =
    new WALFactory(utility.getConfiguration(), URLEncoder.encode("regionserver:60020", "UTF8"));
  ReplicationSourceManager replicationManager = replication.getReplicationManager();
  wals.getWALProvider()
    .addWALActionsListener(new ReplicationSourceWALActionListener(conf, replicationManager));
  final WAL wal = wals.getWAL(hri);
  manager.init();
  TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf("tableame"))
    .setColumnFamily(ColumnFamilyDescriptorBuilder.of(f1)).build();
  NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  for(byte[] fam : htd.getColumnFamilyNames()) {
    scopes.put(fam, 0);
  }
  // Testing normal log rolling every 20
  for(long i = 1; i < 101; i++) {
    if(i > 1 && i % 20 == 0) {
      wal.rollWriter();
    }
    LOG.info(Long.toString(i));
    final long txid = wal.appendData(hri,
      new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes),
      edit);
    wal.sync(txid);
  }

  // Simulate a rapid insert that's followed
  // by a report that's still not totally complete (missing last one)
  LOG.info(baseline + " and " + time);
  baseline += 101;
  time = baseline;
  LOG.info(baseline + " and " + time);

  for (int i = 0; i < 3; i++) {
    wal.appendData(hri,
      new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes),
      edit);
  }
  wal.sync();

  int logNumber = 0;
  for (Map.Entry<String, NavigableSet<String>> entry : manager.getWALs().get(slaveId)
    .entrySet()) {
    logNumber += entry.getValue().size();
  }
  assertEquals(6, logNumber);

  wal.rollWriter();

  ReplicationSourceInterface source = mock(ReplicationSourceInterface.class);
  when(source.getQueueId()).thenReturn("1");
  when(source.isRecovered()).thenReturn(false);
  when(source.isSyncReplication()).thenReturn(false);
  manager.logPositionAndCleanOldLogs(source,
    new WALEntryBatch(0, manager.getSources().get(0).getCurrentPath()));

  wal.appendData(hri,
    new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes),
    edit);
  wal.sync();

  assertEquals(1, manager.getWALs().size());


  // TODO Need a case with only 2 WALs and we only want to delete the first one
}
 
Example 18
Source File: TestPerRegionIndexWriteCache.java    From phoenix with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("deprecation")
@Before
public void setUp() throws Exception {
    Path hbaseRootDir = new Path(getClass().getSimpleName() + "_" + testName.getMethodName());
    TEST_UTIL.getConfiguration().set("hbase.rootdir", hbaseRootDir.toString());

    FileSystem newFS = miniDfs.getFileSystem();
    RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).setStartKey(null).setEndKey(null).setSplit(false).build();
    Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
    Random rn = new Random();
    tableName = TableName.valueOf("TestPerRegion" + rn.nextInt());
    WALFactory walFactory = new WALFactory(TEST_UTIL.getConfiguration(), getClass().getSimpleName());
    wal = walFactory.getWAL(RegionInfoBuilder.newBuilder(TableName.valueOf("logs")).build());
      TableDescriptor htd =
              TableDescriptorBuilder
                      .newBuilder(tableName)
                      .addColumnFamily(
                          ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("a")).build())
                      .build();
    
    r1 = new HRegion(basedir, wal, newFS, TEST_UTIL.getConfiguration(), hri, htd, null) {
        @Override
        public int hashCode() {
          return 1;
        }

        @Override
        public String toString() {
          return "testRegion1";
        }
      };
      
    r2 = new HRegion(basedir, wal, newFS, TEST_UTIL.getConfiguration(), hri, htd, null) {
        @Override
        public int hashCode() {
          return 2;
        }

        @Override
        public String toString() {
          return "testRegion1";
        }
      };
}
 
Example 19
Source File: TestLogRollingNoCluster.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * Spin up a bunch of threads and have them all append to a WAL.  Roll the
 * WAL frequently to try and trigger NPE.
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public void testContendedLogRolling() throws Exception {
  TEST_UTIL.startMiniDFSCluster(3);
  Path dir = TEST_UTIL.getDataTestDirOnTestFS();

  // The implementation needs to know the 'handler' count.
  TEST_UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HANDLER_COUNT, NUM_THREADS);
  final Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
  conf.set(WALFactory.WAL_PROVIDER, "filesystem");
  CommonFSUtils.setRootDir(conf, dir);
  FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(TEST_UTIL.getConfiguration());
  FSTableDescriptors.tryUpdateMetaTableDescriptor(TEST_UTIL.getConfiguration());
  TableDescriptor metaTableDescriptor = fsTableDescriptors.get(TableName.META_TABLE_NAME);
  conf.set("hbase.regionserver.hlog.writer.impl", HighLatencySyncWriter.class.getName());
  final WALFactory wals = new WALFactory(conf, TestLogRollingNoCluster.class.getName());
  final WAL wal = wals.getWAL(null);

  Appender [] appenders = null;

  final int numThreads = NUM_THREADS;
  appenders = new Appender[numThreads];
  try {
    for (int i = 0; i < numThreads; i++) {
      // Have each appending thread write 'count' entries
      appenders[i] = new Appender(metaTableDescriptor, wal, i, NUM_ENTRIES);
    }
    for (int i = 0; i < numThreads; i++) {
      appenders[i].start();
    }
    for (int i = 0; i < numThreads; i++) {
      //ensure that all threads are joined before closing the wal
      appenders[i].join();
    }
  } finally {
    wals.close();
  }
  for (int i = 0; i < numThreads; i++) {
    assertFalse("Error: " + appenders[i].getException(), appenders[i].isException());
  }
  TEST_UTIL.shutdownMiniDFSCluster();
}
 
Example 20
Source File: TestLogRollAbort.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * Tests the case where a RegionServer enters a GC pause,
 * comes back online after the master declared it dead and started to split.
 * Want log rolling after a master split to fail. See HBASE-2312.
 */
@Test
public void testLogRollAfterSplitStart() throws IOException {
  LOG.info("Verify wal roll after split starts will fail.");
  String logName = ServerName.valueOf("testLogRollAfterSplitStart",
      16010, System.currentTimeMillis()).toString();
  Path thisTestsDir = new Path(HBASELOGDIR, AbstractFSWALProvider.getWALDirectoryName(logName));
  final WALFactory wals = new WALFactory(conf, logName);

  try {
    // put some entries in an WAL
    TableName tableName =
        TableName.valueOf(this.getClass().getName());
    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build();
    WAL log = wals.getWAL(regionInfo);
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);

    int total = 20;
    for (int i = 0; i < total; i++) {
      WALEdit kvs = new WALEdit();
      kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));
      NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
      scopes.put(Bytes.toBytes("column"), 0);
      log.appendData(regionInfo, new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName,
        System.currentTimeMillis(), mvcc, scopes), kvs);
    }
    // Send the data to HDFS datanodes and close the HDFS writer
    log.sync();
    ((AbstractFSWAL<?>) log).replaceWriter(((FSHLog)log).getOldPath(), null, null);

    // code taken from MasterFileSystem.getLogDirs(), which is called from
    // MasterFileSystem.splitLog() handles RS shutdowns (as observed by the splitting process)
    // rename the directory so a rogue RS doesn't create more WALs
    Path rsSplitDir = thisTestsDir.suffix(AbstractFSWALProvider.SPLITTING_EXT);
    if (!fs.rename(thisTestsDir, rsSplitDir)) {
      throw new IOException("Failed fs.rename for log split: " + thisTestsDir);
    }
    LOG.debug("Renamed region directory: " + rsSplitDir);

    LOG.debug("Processing the old log files.");
    WALSplitter.split(HBASELOGDIR, rsSplitDir, OLDLOGDIR, fs, conf, wals);

    LOG.debug("Trying to roll the WAL.");
    try {
      log.rollWriter();
      Assert.fail("rollWriter() did not throw any exception.");
    } catch (IOException ioe) {
      if (ioe.getCause() instanceof FileNotFoundException) {
        LOG.info("Got the expected exception: ", ioe.getCause());
      } else {
        Assert.fail("Unexpected exception: " + ioe);
      }
    }
  } finally {
    wals.close();
    if (fs.exists(thisTestsDir)) {
      fs.delete(thisTestsDir, true);
    }
  }
}