org.apache.hadoop.hbase.regionserver.HRegionFileSystem Java Examples

The following examples show how to use org.apache.hadoop.hbase.regionserver.HRegionFileSystem. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1}, null);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example #2
Source File: OfflineMetaRebuildTestCore.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected RegionInfo createRegion(Configuration conf, final Table htbl,
    byte[] startKey, byte[] endKey) throws IOException {
  Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
  RegionInfo hri = RegionInfoBuilder.newBuilder(htbl.getName())
      .setStartKey(startKey)
      .setEndKey(endKey)
      .build();

  LOG.info("manually adding regioninfo and hdfs data: " + hri.toString());
  Path rootDir = CommonFSUtils.getRootDir(conf);
  FileSystem fs = rootDir.getFileSystem(conf);
  Path p = new Path(CommonFSUtils.getTableDir(rootDir, htbl.getName()),
      hri.getEncodedName());
  fs.mkdirs(p);
  Path riPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
  FSDataOutputStream out = fs.create(riPath);
  out.write(RegionInfo.toDelimitedByteArray(hri));
  out.close();

  // add to meta.
  MetaTableAccessor.addRegionToMeta(TEST_UTIL.getConnection(), hri);
  meta.close();
  return hri;
}
 
Example #3
Source File: TestMajorCompactionRequest.java    From hbase with Apache License 2.0 6 votes vote down vote up
private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles,
    List<StoreFileInfo> storeFiles, long referenceFileTimestamp) throws IOException {
  FileSystem fileSystem = mock(FileSystem.class);
  if (hasReferenceFiles) {
    FileStatus fileStatus = mock(FileStatus.class);
    doReturn(referenceFileTimestamp).when(fileStatus).getModificationTime();
    doReturn(fileStatus).when(fileSystem).getFileLinkStatus(isA(Path.class));
  }
  HRegionFileSystem mockSystem = mock(HRegionFileSystem.class);
  doReturn(info).when(mockSystem).getRegionInfo();
  doReturn(regionStoreDir).when(mockSystem).getStoreDir(FAMILY);
  doReturn(hasReferenceFiles).when(mockSystem).hasReferences(anyString());
  doReturn(storeFiles).when(mockSystem).getStoreFiles(anyString());
  doReturn(fileSystem).when(mockSystem).getFileSystem();
  return mockSystem;
}
 
Example #4
Source File: TestMajorCompactionRequest.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test public void testIfWeHaveNewReferenceFilesButOldStoreFiles() throws Exception {
  // this tests that reference files that are new, but have older timestamps for the files
  // they reference still will get compacted.
  TableName table = TableName.valueOf("TestMajorCompactor");
  TableDescriptor htd = UTILITY.createTableDescriptor(table, Bytes.toBytes(FAMILY));
  RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
  HRegion region =
      HBaseTestingUtility.createRegionAndWAL(hri, rootRegionDir, UTILITY.getConfiguration(), htd);

  Configuration configuration = mock(Configuration.class);
  // the reference file timestamp is newer
  List<StoreFileInfo> storeFiles = mockStoreFiles(regionStoreDir, 4, 101);
  List<Path> paths = storeFiles.stream().map(StoreFileInfo::getPath).collect(Collectors.toList());
  // the files that are referenced are older, thus we still compact.
  HRegionFileSystem fileSystem =
      mockFileSystem(region.getRegionInfo(), true, storeFiles, 50);
  MajorCompactionRequest majorCompactionRequest = spy(new MajorCompactionRequest(configuration,
      region.getRegionInfo(), Sets.newHashSet(FAMILY)));
  doReturn(mock(Connection.class)).when(majorCompactionRequest).getConnection(eq(configuration));
  doReturn(paths).when(majorCompactionRequest).getReferenceFilePaths(any(FileSystem.class),
      any(Path.class));
  doReturn(fileSystem).when(majorCompactionRequest).getFileSystem(any(Connection.class));
  Set<String> result =
      majorCompactionRequest.getStoresRequiringCompaction(Sets.newHashSet("a"), 100);
  assertEquals(FAMILY, Iterables.getOnlyElement(result));
}
 
Example #5
Source File: MajorCompactionRequest.java    From hbase with Apache License 2.0 6 votes vote down vote up
boolean shouldCFBeCompacted(HRegionFileSystem fileSystem, String family, long ts)
    throws IOException {

  // do we have any store files?
  Collection<StoreFileInfo> storeFiles = fileSystem.getStoreFiles(family);
  if (storeFiles == null) {
    LOG.info("Excluding store: " + family + " for compaction for region:  " + fileSystem
        .getRegionInfo().getEncodedName(), " has no store files");
    return false;
  }
  // check for reference files
  if (fileSystem.hasReferences(family) && familyHasReferenceFile(fileSystem, family, ts)) {
    LOG.info("Including store: " + family + " with: " + storeFiles.size()
        + " files for compaction for region: " + fileSystem.getRegionInfo().getEncodedName());
    return true;
  }
  // check store file timestamps
  boolean includeStore = this.shouldIncludeStore(fileSystem, family, storeFiles, ts);
  if (!includeStore) {
    LOG.info("Excluding store: " + family + " for compaction for region:  " + fileSystem
        .getRegionInfo().getEncodedName() + " already compacted");
  }
  return includeStore;
}
 
Example #6
Source File: MajorCompactionTTLRequest.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
protected boolean shouldIncludeStore(HRegionFileSystem fileSystem, String family,
    Collection<StoreFileInfo> storeFiles, long ts) throws IOException {

  for (StoreFileInfo storeFile : storeFiles) {
    // Lets only compact when all files are older than TTL
    if (storeFile.getModificationTime() >= ts) {
      LOG.info("There is atleast one file in store: " + family + " file: " + storeFile.getPath()
          + " with timestamp " + storeFile.getModificationTime()
          + " for region: " + fileSystem.getRegionInfo().getEncodedName()
          + " older than TTL: " + ts);
      return false;
    }
  }
  return true;
}
 
Example #7
Source File: RestoreSnapshotHelper.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * @return the set of the regions contained in the table
 */
private List<RegionInfo> getTableRegions() throws IOException {
  LOG.debug("get table regions: " + tableDir);
  FileStatus[] regionDirs =
    CommonFSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
  if (regionDirs == null) {
    return null;
  }

  List<RegionInfo> regions = new ArrayList<>(regionDirs.length);
  for (int i = 0; i < regionDirs.length; ++i) {
    RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDirs[i].getPath());
    regions.add(hri);
  }
  LOG.debug("found " + regions.size() + " regions for table=" +
    tableDesc.getTableName().getNameAsString());
  return regions;
}
 
Example #8
Source File: SplitTableRegionProcedure.java    From hbase with Apache License 2.0 6 votes vote down vote up
private Pair<Path, Path> splitStoreFile(HRegionFileSystem regionFs, byte[] family, HStoreFile sf)
  throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("pid=" + getProcId() + " splitting started for store file: " +
        sf.getPath() + " for region: " + getParentRegion().getShortNameToLog());
  }

  final byte[] splitRow = getSplitRow();
  final String familyName = Bytes.toString(family);
  final Path path_first = regionFs.splitStoreFile(this.daughterOneRI, familyName, sf, splitRow,
      false, splitPolicy);
  final Path path_second = regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf, splitRow,
     true, splitPolicy);
  if (LOG.isDebugEnabled()) {
    LOG.debug("pid=" + getProcId() + " splitting complete for store file: " +
        sf.getPath() + " for region: " + getParentRegion().getShortNameToLog());
  }
  return new Pair<Path,Path>(path_first, path_second);
}
 
Example #9
Source File: SplitTableRegionProcedure.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Create daughter regions
 */
@VisibleForTesting
public void createDaughterRegions(final MasterProcedureEnv env) throws IOException {
  final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
  final Path tabledir = CommonFSUtils.getTableDir(mfs.getRootDir(), getTableName());
  final FileSystem fs = mfs.getFileSystem();
  HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
    env.getMasterConfiguration(), fs, tabledir, getParentRegion(), false);
  regionFs.createSplitsDir(daughterOneRI, daughterTwoRI);

  Pair<Integer, Integer> expectedReferences = splitStoreFiles(env, regionFs);

  assertReferenceFileCount(fs, expectedReferences.getFirst(),
    regionFs.getSplitsDir(daughterOneRI));
  //Move the files from the temporary .splits to the final /table/region directory
  regionFs.commitDaughterRegion(daughterOneRI);
  assertReferenceFileCount(fs, expectedReferences.getFirst(),
    new Path(tabledir, daughterOneRI.getEncodedName()));

  assertReferenceFileCount(fs, expectedReferences.getSecond(),
    regionFs.getSplitsDir(daughterTwoRI));
  regionFs.commitDaughterRegion(daughterTwoRI);
  assertReferenceFileCount(fs, expectedReferences.getSecond(),
    new Path(tabledir, daughterTwoRI.getEncodedName()));
}
 
Example #10
Source File: MergeTableRegionsProcedure.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Create reference file(s) to parent region hfiles in the <code>mergeDir</code>
 * @param regionFs merge parent region file system
 * @param mergeDir the temp directory in which we are accumulating references.
 */
private void mergeStoreFiles(final MasterProcedureEnv env, final HRegionFileSystem regionFs,
    final Path mergeDir) throws IOException {
  final TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(getTableName());
  for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
    String family = hcd.getNameAsString();
    final Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(family);
    if (storeFiles != null && storeFiles.size() > 0) {
      for (StoreFileInfo storeFileInfo : storeFiles) {
        // Create reference file(s) to parent region file here in mergedDir.
        // As this procedure is running on master, use CacheConfig.DISABLED means
        // don't cache any block.
        regionFs.mergeStoreFile(mergedRegion, family, new HStoreFile(
            storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED), mergeDir);
      }
    }
  }
}
 
Example #11
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  Path hlogPath = new Path(FSUtils.getRootDir(conf) + "/hlog");
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  HLog hLog = HLogFactory.createHLog(fs, hlogPath, tableName, conf);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd, new MockRegionServerServices(conf, null));
}
 
Example #12
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example #13
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  Path hlogPath = new Path(FSUtils.getRootDir(conf) + "/hlog");
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  HLog hLog = HLogFactory.createHLog(fs, hlogPath, tableName, conf);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd, new MockRegionServerServices(conf, null));
}
 
Example #14
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1}, null);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example #15
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, tableName + ".hlog");
  HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
  WAL hLog = walFactory.getWAL(info);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example #16
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
                     new LocalRegionServerServices(conf, ServerName.valueOf(
                       InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example #17
Source File: TransactionProcessorTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
                     new LocalRegionServerServices(conf, ServerName.valueOf(
                       InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example #18
Source File: CatalogJanitor.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * If merged region no longer holds reference to the merge regions, archive
 * merge region on hdfs and perform deleting references in hbase:meta
 * @return true if we delete references in merged region on hbase:meta and archive
 *   the files on the file system
 */
private boolean cleanMergeRegion(final RegionInfo mergedRegion, List<RegionInfo> parents)
    throws IOException {
  FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
  Path rootdir = this.services.getMasterFileSystem().getRootDir();
  Path tabledir = CommonFSUtils.getTableDir(rootdir, mergedRegion.getTable());
  TableDescriptor htd = getDescriptor(mergedRegion.getTable());
  HRegionFileSystem regionFs = null;
  try {
    regionFs = HRegionFileSystem.openRegionFromFileSystem(
        this.services.getConfiguration(), fs, tabledir, mergedRegion, true);
  } catch (IOException e) {
    LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());
  }
  if (regionFs == null || !regionFs.hasReferences(htd)) {
    LOG.debug("Deleting parents ({}) from fs; merged child {} no longer holds references",
         parents.stream().map(r -> RegionInfo.getShortNameToLog(r)).
            collect(Collectors.joining(", ")),
        mergedRegion);
    ProcedureExecutor<MasterProcedureEnv> pe = this.services.getMasterProcedureExecutor();
    pe.submitProcedure(new GCMultipleMergedRegionsProcedure(pe.getEnvironment(),
        mergedRegion,  parents));
    for (RegionInfo ri:  parents) {
      // The above scheduled GCMultipleMergedRegionsProcedure does the below.
      // Do we need this?
      this.services.getAssignmentManager().getRegionStates().deleteRegion(ri);
      this.services.getServerManager().removeRegion(ri);
    }
    return true;
  }
  return false;
}
 
Example #19
Source File: TestMajorCompactionTTLRequest.java    From hbase with Apache License 2.0 5 votes vote down vote up
private MajorCompactionTTLRequest makeMockRequest(List<StoreFileInfo> storeFiles)
    throws IOException {
  Configuration configuration = mock(Configuration.class);
  RegionInfo regionInfo = mock(RegionInfo.class);
  when(regionInfo.getEncodedName()).thenReturn("HBase");
  when(regionInfo.getTable()).thenReturn(TableName.valueOf("foo"));
  MajorCompactionTTLRequest request = new MajorCompactionTTLRequest(configuration, regionInfo);
  MajorCompactionTTLRequest spy = spy(request);
  HRegionFileSystem fileSystem = mockFileSystem(regionInfo, false, storeFiles);
  doReturn(fileSystem).when(spy).getFileSystem(isA(Connection.class));
  doReturn(mock(Connection.class)).when(spy).getConnection(eq(configuration));
  return spy;
}
 
Example #20
Source File: TestMajorCompactionRequest.java    From hbase with Apache License 2.0 5 votes vote down vote up
private MajorCompactionRequest makeMockRequest(List<StoreFileInfo> storeFiles,
    boolean references) throws IOException {
  Configuration configuration = mock(Configuration.class);
  RegionInfo regionInfo = mock(RegionInfo.class);
  when(regionInfo.getEncodedName()).thenReturn("HBase");
  when(regionInfo.getTable()).thenReturn(TableName.valueOf("foo"));
  MajorCompactionRequest request =
      new MajorCompactionRequest(configuration, regionInfo, Sets.newHashSet("a"));
  MajorCompactionRequest spy = spy(request);
  HRegionFileSystem fileSystem = mockFileSystem(regionInfo, references, storeFiles);
  doReturn(fileSystem).when(spy).getFileSystem(isA(Connection.class));
  doReturn(mock(Connection.class)).when(spy).getConnection(eq(configuration));
  return spy;
}
 
Example #21
Source File: SnapshotTestingUtils.java    From hbase with Apache License 2.0 5 votes vote down vote up
private RegionData[] createTable(final TableDescriptor htd, final int nregions)
    throws IOException {
  Path tableDir = CommonFSUtils.getTableDir(rootDir, htd.getTableName());
  new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);

  assertTrue(nregions % 2 == 0);
  RegionData[] regions = new RegionData[nregions];
  for (int i = 0; i < regions.length; i += 2) {
    byte[] startKey = Bytes.toBytes(0 + i * 2);
    byte[] endKey = Bytes.toBytes(1 + i * 2);

    // First region, simple with one plain hfile.
    RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName())
        .setStartKey(startKey)
        .setEndKey(endKey)
        .build();
    HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
    regions[i] = new RegionData(tableDir, hri, 3);
    for (int j = 0; j < regions[i].files.length; ++j) {
      Path storeFile = createStoreFile(rfs.createTempName());
      regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
    }

    // Second region, used to test the split case.
    // This region contains a reference to the hfile in the first region.
    startKey = Bytes.toBytes(2 + i * 2);
    endKey = Bytes.toBytes(3 + i * 2);
    hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
    rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
    regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
    for (int j = 0; j < regions[i].files.length; ++j) {
      String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
      Path refFile = createStoreFile(new Path(rootDir, refName));
      regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
    }
  }
  return regions;
}
 
Example #22
Source File: MajorCompactionRequest.java    From hbase with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
HRegionFileSystem getFileSystem(Connection connection) throws IOException {
  Admin admin = connection.getAdmin();
  return HRegionFileSystem.openRegionFromFileSystem(admin.getConfiguration(),
    CommonFSUtils.getCurrentFileSystem(admin.getConfiguration()), CommonFSUtils.getTableDir(
      CommonFSUtils.getRootDir(admin.getConfiguration()), region.getTable()),
    region, true);
}
 
Example #23
Source File: MajorCompactionRequest.java    From hbase with Apache License 2.0 5 votes vote down vote up
protected boolean familyHasReferenceFile(HRegionFileSystem fileSystem, String family, long ts)
    throws IOException {
  List<Path> referenceFiles =
      getReferenceFilePaths(fileSystem.getFileSystem(), fileSystem.getStoreDir(family));
  for (Path referenceFile : referenceFiles) {
    FileStatus status = fileSystem.getFileSystem().getFileLinkStatus(referenceFile);
    if (status.getModificationTime() < ts) {
      LOG.info("Including store: " + family + " for compaction for region:  " + fileSystem
          .getRegionInfo().getEncodedName() + " (reference store files)");
      return true;
    }
  }
  return false;

}
 
Example #24
Source File: MajorCompactionRequest.java    From hbase with Apache License 2.0 5 votes vote down vote up
protected boolean shouldIncludeStore(HRegionFileSystem fileSystem, String family,
    Collection<StoreFileInfo> storeFiles, long ts) throws IOException {

  for (StoreFileInfo storeFile : storeFiles) {
    if (storeFile.getModificationTime() < ts) {
      LOG.info("Including store: " + family + " with: " + storeFiles.size()
          + " files for compaction for region: "
          + fileSystem.getRegionInfo().getEncodedName());
      return true;
    }
  }
  return false;
}
 
Example #25
Source File: MajorCompactionRequest.java    From hbase with Apache License 2.0 5 votes vote down vote up
Set<String> getStoresRequiringCompaction(Set<String> requestedStores, long timestamp)
    throws IOException {
  try(Connection connection = getConnection(configuration)) {
    HRegionFileSystem fileSystem = getFileSystem(connection);
    Set<String> familiesToCompact = Sets.newHashSet();
    for (String family : requestedStores) {
      if (shouldCFBeCompacted(fileSystem, family, timestamp)) {
        familiesToCompact.add(family);
      }
    }
    return familiesToCompact;
  }
}
 
Example #26
Source File: MajorCompactionTTLRequest.java    From hbase with Apache License 2.0 5 votes vote down vote up
Map<String, Long> getStoresRequiringCompaction(TableDescriptor htd) throws IOException {
  try(Connection connection = getConnection(configuration)) {
    HRegionFileSystem fileSystem = getFileSystem(connection);
    Map<String, Long> familyTTLMap = Maps.newHashMap();
    for (ColumnFamilyDescriptor descriptor : htd.getColumnFamilies()) {
      long ts = getColFamilyCutoffTime(descriptor);
      // If the table's TTL is forever, lets not compact any of the regions.
      if (ts > 0 && shouldCFBeCompacted(fileSystem, descriptor.getNameAsString(), ts)) {
        familyTTLMap.put(descriptor.getNameAsString(), ts);
      }
    }
    return familyTTLMap;
  }
}
 
Example #27
Source File: MergeTableRegionsProcedure.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Clean up a merged region on rollback after failure.
 */
private void cleanupMergedRegion(final MasterProcedureEnv env) throws IOException {
  final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
  TableName tn = this.regionsToMerge[0].getTable();
  final Path tabledir = CommonFSUtils.getTableDir(mfs.getRootDir(), tn);
  final FileSystem fs = mfs.getFileSystem();
  // See createMergedRegion above where we specify the merge dir as being in the
  // FIRST merge parent region.
  HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
    env.getMasterConfiguration(), fs, tabledir, regionsToMerge[0], false);
  regionFs.cleanupMergedRegion(mergedRegion);
}
 
Example #28
Source File: SnapshotManifestV1.java    From hbase with Apache License 2.0 4 votes vote down vote up
static SnapshotRegionManifest buildManifestFromDisk(final Configuration conf,
    final FileSystem fs, final Path tableDir, final RegionInfo regionInfo) throws IOException {
  HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs,
        tableDir, regionInfo, true);
  SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();

  // 1. dump region meta info into the snapshot directory
  LOG.debug("Storing region-info for snapshot.");
  manifest.setRegionInfo(ProtobufUtil.toRegionInfo(regionInfo));

  // 2. iterate through all the stores in the region
  LOG.debug("Creating references for hfiles");

  // This ensures that we have an atomic view of the directory as long as we have < ls limit
  // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
  // batches and may miss files being added/deleted. This could be more robust (iteratively
  // checking to see if we have all the files until we are sure), but the limit is currently 1000
  // files/batch, far more than the number of store files under a single column family.
  Collection<String> familyNames = regionFs.getFamilies();
  if (familyNames != null) {
    for (String familyName: familyNames) {
      Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName, false);
      if (storeFiles == null) {
        LOG.debug("No files under family: " + familyName);
        continue;
      }

      // 2.1. build the snapshot reference for the store
      SnapshotRegionManifest.FamilyFiles.Builder family =
            SnapshotRegionManifest.FamilyFiles.newBuilder();
      family.setFamilyName(UnsafeByteOperations.unsafeWrap(Bytes.toBytes(familyName)));

      if (LOG.isDebugEnabled()) {
        LOG.debug("Adding snapshot references for " + storeFiles  + " hfiles");
      }

      // 2.2. iterate through all the store's files and create "references".
      int i = 0;
      int sz = storeFiles.size();
      for (StoreFileInfo storeFile: storeFiles) {
        // create "reference" to this store file.
        LOG.debug("Adding reference for file ("+ (++i) +"/" + sz + "): " + storeFile.getPath());
        SnapshotRegionManifest.StoreFile.Builder sfManifest =
              SnapshotRegionManifest.StoreFile.newBuilder();
        sfManifest.setName(storeFile.getPath().getName());
        family.addStoreFiles(sfManifest.build());
      }
      manifest.addFamilyFiles(family.build());
    }
  }
  return manifest.build();
}
 
Example #29
Source File: SnapshotManifestV1.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public void familyClose(final HRegionFileSystem region, final Path family) {
}
 
Example #30
Source File: SnapshotManifestV1.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public Path familyOpen(final HRegionFileSystem snapshotRegionFs, final byte[] familyName) {
  Path familyDir = snapshotRegionFs.getStoreDir(Bytes.toString(familyName));
  return familyDir;
}