Java Code Examples for org.apache.hadoop.hbase.regionserver.HRegionFileSystem

The following examples show how to use org.apache.hadoop.hbase.regionserver.HRegionFileSystem. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1}, null);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example 2
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  Path hlogPath = new Path(FSUtils.getRootDir(conf) + "/hlog");
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  HLog hLog = HLogFactory.createHLog(fs, hlogPath, tableName, conf);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd, new MockRegionServerServices(conf, null));
}
 
Example 3
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1}, null);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example 4
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, tableName + ".hlog");
  HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
  WAL hLog = walFactory.getWAL(info);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example 5
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
                     new LocalRegionServerServices(conf, ServerName.valueOf(
                       InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example 6
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
                     new LocalRegionServerServices(conf, ServerName.valueOf(
                       InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example 7
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1});
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example 8
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  Path hlogPath = new Path(FSUtils.getRootDir(conf) + "/hlog");
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  HLog hLog = HLogFactory.createHLog(fs, hlogPath, tableName, conf);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd, new MockRegionServerServices(conf, null));
}
 
Example 9
Source Project: hbase   Source File: MergeTableRegionsProcedure.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Create reference file(s) to parent region hfiles in the <code>mergeDir</code>
 * @param regionFs merge parent region file system
 * @param mergeDir the temp directory in which we are accumulating references.
 */
private void mergeStoreFiles(final MasterProcedureEnv env, final HRegionFileSystem regionFs,
    final Path mergeDir) throws IOException {
  final TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(getTableName());
  for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
    String family = hcd.getNameAsString();
    final Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(family);
    if (storeFiles != null && storeFiles.size() > 0) {
      for (StoreFileInfo storeFileInfo : storeFiles) {
        // Create reference file(s) to parent region file here in mergedDir.
        // As this procedure is running on master, use CacheConfig.DISABLED means
        // don't cache any block.
        regionFs.mergeStoreFile(mergedRegion, family, new HStoreFile(
            storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED), mergeDir);
      }
    }
  }
}
 
Example 10
Source Project: hbase   Source File: SplitTableRegionProcedure.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Create daughter regions
 */
@VisibleForTesting
public void createDaughterRegions(final MasterProcedureEnv env) throws IOException {
  final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
  final Path tabledir = CommonFSUtils.getTableDir(mfs.getRootDir(), getTableName());
  final FileSystem fs = mfs.getFileSystem();
  HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
    env.getMasterConfiguration(), fs, tabledir, getParentRegion(), false);
  regionFs.createSplitsDir(daughterOneRI, daughterTwoRI);

  Pair<Integer, Integer> expectedReferences = splitStoreFiles(env, regionFs);

  assertReferenceFileCount(fs, expectedReferences.getFirst(),
    regionFs.getSplitsDir(daughterOneRI));
  //Move the files from the temporary .splits to the final /table/region directory
  regionFs.commitDaughterRegion(daughterOneRI);
  assertReferenceFileCount(fs, expectedReferences.getFirst(),
    new Path(tabledir, daughterOneRI.getEncodedName()));

  assertReferenceFileCount(fs, expectedReferences.getSecond(),
    regionFs.getSplitsDir(daughterTwoRI));
  regionFs.commitDaughterRegion(daughterTwoRI);
  assertReferenceFileCount(fs, expectedReferences.getSecond(),
    new Path(tabledir, daughterTwoRI.getEncodedName()));
}
 
Example 11
Source Project: hbase   Source File: SplitTableRegionProcedure.java    License: Apache License 2.0 6 votes vote down vote up
private Pair<Path, Path> splitStoreFile(HRegionFileSystem regionFs, byte[] family, HStoreFile sf)
  throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("pid=" + getProcId() + " splitting started for store file: " +
        sf.getPath() + " for region: " + getParentRegion().getShortNameToLog());
  }

  final byte[] splitRow = getSplitRow();
  final String familyName = Bytes.toString(family);
  final Path path_first = regionFs.splitStoreFile(this.daughterOneRI, familyName, sf, splitRow,
      false, splitPolicy);
  final Path path_second = regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf, splitRow,
     true, splitPolicy);
  if (LOG.isDebugEnabled()) {
    LOG.debug("pid=" + getProcId() + " splitting complete for store file: " +
        sf.getPath() + " for region: " + getParentRegion().getShortNameToLog());
  }
  return new Pair<Path,Path>(path_first, path_second);
}
 
Example 12
Source Project: hbase   Source File: RestoreSnapshotHelper.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @return the set of the regions contained in the table
 */
private List<RegionInfo> getTableRegions() throws IOException {
  LOG.debug("get table regions: " + tableDir);
  FileStatus[] regionDirs =
    CommonFSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
  if (regionDirs == null) {
    return null;
  }

  List<RegionInfo> regions = new ArrayList<>(regionDirs.length);
  for (int i = 0; i < regionDirs.length; ++i) {
    RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDirs[i].getPath());
    regions.add(hri);
  }
  LOG.debug("found " + regions.size() + " regions for table=" +
    tableDesc.getTableName().getNameAsString());
  return regions;
}
 
Example 13
Source Project: hbase   Source File: MajorCompactionTTLRequest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected boolean shouldIncludeStore(HRegionFileSystem fileSystem, String family,
    Collection<StoreFileInfo> storeFiles, long ts) throws IOException {

  for (StoreFileInfo storeFile : storeFiles) {
    // Lets only compact when all files are older than TTL
    if (storeFile.getModificationTime() >= ts) {
      LOG.info("There is atleast one file in store: " + family + " file: " + storeFile.getPath()
          + " with timestamp " + storeFile.getModificationTime()
          + " for region: " + fileSystem.getRegionInfo().getEncodedName()
          + " older than TTL: " + ts);
      return false;
    }
  }
  return true;
}
 
Example 14
Source Project: hbase   Source File: MajorCompactionRequest.java    License: Apache License 2.0 6 votes vote down vote up
boolean shouldCFBeCompacted(HRegionFileSystem fileSystem, String family, long ts)
    throws IOException {

  // do we have any store files?
  Collection<StoreFileInfo> storeFiles = fileSystem.getStoreFiles(family);
  if (storeFiles == null) {
    LOG.info("Excluding store: " + family + " for compaction for region:  " + fileSystem
        .getRegionInfo().getEncodedName(), " has no store files");
    return false;
  }
  // check for reference files
  if (fileSystem.hasReferences(family) && familyHasReferenceFile(fileSystem, family, ts)) {
    LOG.info("Including store: " + family + " with: " + storeFiles.size()
        + " files for compaction for region: " + fileSystem.getRegionInfo().getEncodedName());
    return true;
  }
  // check store file timestamps
  boolean includeStore = this.shouldIncludeStore(fileSystem, family, storeFiles, ts);
  if (!includeStore) {
    LOG.info("Excluding store: " + family + " for compaction for region:  " + fileSystem
        .getRegionInfo().getEncodedName() + " already compacted");
  }
  return includeStore;
}
 
Example 15
Source Project: hbase   Source File: TestMajorCompactionRequest.java    License: Apache License 2.0 6 votes vote down vote up
@Test public void testIfWeHaveNewReferenceFilesButOldStoreFiles() throws Exception {
  // this tests that reference files that are new, but have older timestamps for the files
  // they reference still will get compacted.
  TableName table = TableName.valueOf("TestMajorCompactor");
  TableDescriptor htd = UTILITY.createTableDescriptor(table, Bytes.toBytes(FAMILY));
  RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
  HRegion region =
      HBaseTestingUtility.createRegionAndWAL(hri, rootRegionDir, UTILITY.getConfiguration(), htd);

  Configuration configuration = mock(Configuration.class);
  // the reference file timestamp is newer
  List<StoreFileInfo> storeFiles = mockStoreFiles(regionStoreDir, 4, 101);
  List<Path> paths = storeFiles.stream().map(StoreFileInfo::getPath).collect(Collectors.toList());
  // the files that are referenced are older, thus we still compact.
  HRegionFileSystem fileSystem =
      mockFileSystem(region.getRegionInfo(), true, storeFiles, 50);
  MajorCompactionRequest majorCompactionRequest = spy(new MajorCompactionRequest(configuration,
      region.getRegionInfo(), Sets.newHashSet(FAMILY)));
  doReturn(mock(Connection.class)).when(majorCompactionRequest).getConnection(eq(configuration));
  doReturn(paths).when(majorCompactionRequest).getReferenceFilePaths(any(FileSystem.class),
      any(Path.class));
  doReturn(fileSystem).when(majorCompactionRequest).getFileSystem(any(Connection.class));
  Set<String> result =
      majorCompactionRequest.getStoresRequiringCompaction(Sets.newHashSet("a"), 100);
  assertEquals(FAMILY, Iterables.getOnlyElement(result));
}
 
Example 16
Source Project: hbase   Source File: TestMajorCompactionRequest.java    License: Apache License 2.0 6 votes vote down vote up
private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles,
    List<StoreFileInfo> storeFiles, long referenceFileTimestamp) throws IOException {
  FileSystem fileSystem = mock(FileSystem.class);
  if (hasReferenceFiles) {
    FileStatus fileStatus = mock(FileStatus.class);
    doReturn(referenceFileTimestamp).when(fileStatus).getModificationTime();
    doReturn(fileStatus).when(fileSystem).getFileLinkStatus(isA(Path.class));
  }
  HRegionFileSystem mockSystem = mock(HRegionFileSystem.class);
  doReturn(info).when(mockSystem).getRegionInfo();
  doReturn(regionStoreDir).when(mockSystem).getStoreDir(FAMILY);
  doReturn(hasReferenceFiles).when(mockSystem).hasReferences(anyString());
  doReturn(storeFiles).when(mockSystem).getStoreFiles(anyString());
  doReturn(fileSystem).when(mockSystem).getFileSystem();
  return mockSystem;
}
 
Example 17
Source Project: hbase   Source File: OfflineMetaRebuildTestCore.java    License: Apache License 2.0 6 votes vote down vote up
protected RegionInfo createRegion(Configuration conf, final Table htbl,
    byte[] startKey, byte[] endKey) throws IOException {
  Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
  RegionInfo hri = RegionInfoBuilder.newBuilder(htbl.getName())
      .setStartKey(startKey)
      .setEndKey(endKey)
      .build();

  LOG.info("manually adding regioninfo and hdfs data: " + hri.toString());
  Path rootDir = CommonFSUtils.getRootDir(conf);
  FileSystem fs = rootDir.getFileSystem(conf);
  Path p = new Path(CommonFSUtils.getTableDir(rootDir, htbl.getName()),
      hri.getEncodedName());
  fs.mkdirs(p);
  Path riPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
  FSDataOutputStream out = fs.create(riPath);
  out.write(RegionInfo.toDelimitedByteArray(hri));
  out.close();

  // add to meta.
  MetaTableAccessor.addRegionToMeta(TEST_UTIL.getConnection(), hri);
  meta.close();
  return hri;
}
 
Example 18
Source Project: hbase   Source File: CatalogJanitor.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * If merged region no longer holds reference to the merge regions, archive
 * merge region on hdfs and perform deleting references in hbase:meta
 * @return true if we delete references in merged region on hbase:meta and archive
 *   the files on the file system
 */
private boolean cleanMergeRegion(final RegionInfo mergedRegion, List<RegionInfo> parents)
    throws IOException {
  FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
  Path rootdir = this.services.getMasterFileSystem().getRootDir();
  Path tabledir = CommonFSUtils.getTableDir(rootdir, mergedRegion.getTable());
  TableDescriptor htd = getDescriptor(mergedRegion.getTable());
  HRegionFileSystem regionFs = null;
  try {
    regionFs = HRegionFileSystem.openRegionFromFileSystem(
        this.services.getConfiguration(), fs, tabledir, mergedRegion, true);
  } catch (IOException e) {
    LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());
  }
  if (regionFs == null || !regionFs.hasReferences(htd)) {
    LOG.debug("Deleting parents ({}) from fs; merged child {} no longer holds references",
         parents.stream().map(r -> RegionInfo.getShortNameToLog(r)).
            collect(Collectors.joining(", ")),
        mergedRegion);
    ProcedureExecutor<MasterProcedureEnv> pe = this.services.getMasterProcedureExecutor();
    pe.submitProcedure(new GCMultipleMergedRegionsProcedure(pe.getEnvironment(),
        mergedRegion,  parents));
    for (RegionInfo ri:  parents) {
      // The above scheduled GCMultipleMergedRegionsProcedure does the below.
      // Do we need this?
      this.services.getAssignmentManager().getRegionStates().deleteRegion(ri);
      this.services.getServerManager().removeRegion(ri);
    }
    return true;
  }
  return false;
}
 
Example 19
Source Project: hbase   Source File: MergeTableRegionsProcedure.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Clean up a merged region on rollback after failure.
 */
private void cleanupMergedRegion(final MasterProcedureEnv env) throws IOException {
  final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
  TableName tn = this.regionsToMerge[0].getTable();
  final Path tabledir = CommonFSUtils.getTableDir(mfs.getRootDir(), tn);
  final FileSystem fs = mfs.getFileSystem();
  // See createMergedRegion above where we specify the merge dir as being in the
  // FIRST merge parent region.
  HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
    env.getMasterConfiguration(), fs, tabledir, regionsToMerge[0], false);
  regionFs.cleanupMergedRegion(mergedRegion);
}
 
Example 20
Source Project: hbase   Source File: MajorCompactionTTLRequest.java    License: Apache License 2.0 5 votes vote down vote up
Map<String, Long> getStoresRequiringCompaction(TableDescriptor htd) throws IOException {
  try(Connection connection = getConnection(configuration)) {
    HRegionFileSystem fileSystem = getFileSystem(connection);
    Map<String, Long> familyTTLMap = Maps.newHashMap();
    for (ColumnFamilyDescriptor descriptor : htd.getColumnFamilies()) {
      long ts = getColFamilyCutoffTime(descriptor);
      // If the table's TTL is forever, lets not compact any of the regions.
      if (ts > 0 && shouldCFBeCompacted(fileSystem, descriptor.getNameAsString(), ts)) {
        familyTTLMap.put(descriptor.getNameAsString(), ts);
      }
    }
    return familyTTLMap;
  }
}
 
Example 21
Source Project: hbase   Source File: MajorCompactionRequest.java    License: Apache License 2.0 5 votes vote down vote up
Set<String> getStoresRequiringCompaction(Set<String> requestedStores, long timestamp)
    throws IOException {
  try(Connection connection = getConnection(configuration)) {
    HRegionFileSystem fileSystem = getFileSystem(connection);
    Set<String> familiesToCompact = Sets.newHashSet();
    for (String family : requestedStores) {
      if (shouldCFBeCompacted(fileSystem, family, timestamp)) {
        familiesToCompact.add(family);
      }
    }
    return familiesToCompact;
  }
}
 
Example 22
Source Project: hbase   Source File: MajorCompactionRequest.java    License: Apache License 2.0 5 votes vote down vote up
protected boolean shouldIncludeStore(HRegionFileSystem fileSystem, String family,
    Collection<StoreFileInfo> storeFiles, long ts) throws IOException {

  for (StoreFileInfo storeFile : storeFiles) {
    if (storeFile.getModificationTime() < ts) {
      LOG.info("Including store: " + family + " with: " + storeFiles.size()
          + " files for compaction for region: "
          + fileSystem.getRegionInfo().getEncodedName());
      return true;
    }
  }
  return false;
}
 
Example 23
Source Project: hbase   Source File: MajorCompactionRequest.java    License: Apache License 2.0 5 votes vote down vote up
protected boolean familyHasReferenceFile(HRegionFileSystem fileSystem, String family, long ts)
    throws IOException {
  List<Path> referenceFiles =
      getReferenceFilePaths(fileSystem.getFileSystem(), fileSystem.getStoreDir(family));
  for (Path referenceFile : referenceFiles) {
    FileStatus status = fileSystem.getFileSystem().getFileLinkStatus(referenceFile);
    if (status.getModificationTime() < ts) {
      LOG.info("Including store: " + family + " for compaction for region:  " + fileSystem
          .getRegionInfo().getEncodedName() + " (reference store files)");
      return true;
    }
  }
  return false;

}
 
Example 24
Source Project: hbase   Source File: MajorCompactionRequest.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
HRegionFileSystem getFileSystem(Connection connection) throws IOException {
  Admin admin = connection.getAdmin();
  return HRegionFileSystem.openRegionFromFileSystem(admin.getConfiguration(),
    CommonFSUtils.getCurrentFileSystem(admin.getConfiguration()), CommonFSUtils.getTableDir(
      CommonFSUtils.getRootDir(admin.getConfiguration()), region.getTable()),
    region, true);
}
 
Example 25
Source Project: hbase   Source File: SnapshotTestingUtils.java    License: Apache License 2.0 5 votes vote down vote up
private RegionData[] createTable(final TableDescriptor htd, final int nregions)
    throws IOException {
  Path tableDir = CommonFSUtils.getTableDir(rootDir, htd.getTableName());
  new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);

  assertTrue(nregions % 2 == 0);
  RegionData[] regions = new RegionData[nregions];
  for (int i = 0; i < regions.length; i += 2) {
    byte[] startKey = Bytes.toBytes(0 + i * 2);
    byte[] endKey = Bytes.toBytes(1 + i * 2);

    // First region, simple with one plain hfile.
    RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName())
        .setStartKey(startKey)
        .setEndKey(endKey)
        .build();
    HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
    regions[i] = new RegionData(tableDir, hri, 3);
    for (int j = 0; j < regions[i].files.length; ++j) {
      Path storeFile = createStoreFile(rfs.createTempName());
      regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
    }

    // Second region, used to test the split case.
    // This region contains a reference to the hfile in the first region.
    startKey = Bytes.toBytes(2 + i * 2);
    endKey = Bytes.toBytes(3 + i * 2);
    hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
    rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
    regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
    for (int j = 0; j < regions[i].files.length; ++j) {
      String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
      Path refFile = createStoreFile(new Path(rootDir, refName));
      regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
    }
  }
  return regions;
}
 
Example 26
Source Project: hbase   Source File: TestMajorCompactionRequest.java    License: Apache License 2.0 5 votes vote down vote up
private MajorCompactionRequest makeMockRequest(List<StoreFileInfo> storeFiles,
    boolean references) throws IOException {
  Configuration configuration = mock(Configuration.class);
  RegionInfo regionInfo = mock(RegionInfo.class);
  when(regionInfo.getEncodedName()).thenReturn("HBase");
  when(regionInfo.getTable()).thenReturn(TableName.valueOf("foo"));
  MajorCompactionRequest request =
      new MajorCompactionRequest(configuration, regionInfo, Sets.newHashSet("a"));
  MajorCompactionRequest spy = spy(request);
  HRegionFileSystem fileSystem = mockFileSystem(regionInfo, references, storeFiles);
  doReturn(fileSystem).when(spy).getFileSystem(isA(Connection.class));
  doReturn(mock(Connection.class)).when(spy).getConnection(eq(configuration));
  return spy;
}
 
Example 27
Source Project: hbase   Source File: TestMajorCompactionTTLRequest.java    License: Apache License 2.0 5 votes vote down vote up
private MajorCompactionTTLRequest makeMockRequest(List<StoreFileInfo> storeFiles)
    throws IOException {
  Configuration configuration = mock(Configuration.class);
  RegionInfo regionInfo = mock(RegionInfo.class);
  when(regionInfo.getEncodedName()).thenReturn("HBase");
  when(regionInfo.getTable()).thenReturn(TableName.valueOf("foo"));
  MajorCompactionTTLRequest request = new MajorCompactionTTLRequest(configuration, regionInfo);
  MajorCompactionTTLRequest spy = spy(request);
  HRegionFileSystem fileSystem = mockFileSystem(regionInfo, false, storeFiles);
  doReturn(fileSystem).when(spy).getFileSystem(isA(Connection.class));
  doReturn(mock(Connection.class)).when(spy).getConnection(eq(configuration));
  return spy;
}
 
Example 28
void putRegionInfoFromHdfsInMeta(Path region) throws IOException {
  RegionInfo info = HRegionFileSystem.loadRegionInfoFileContent(fs, region);
  HBCKMetaTableAccessor.addRegionToMeta(conn, info);
}
 
Example 29
Source Project: hbase   Source File: CatalogJanitor.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Checks if a daughter region -- either splitA or splitB -- still holds
 * references to parent.
 * @param parent Parent region
 * @param daughter Daughter region
 * @return A pair where the first boolean says whether or not the daughter
 *   region directory exists in the filesystem and then the second boolean says
 *   whether the daughter has references to the parent.
 */
private Pair<Boolean, Boolean> checkDaughterInFs(final RegionInfo parent,
  final RegionInfo daughter)
throws IOException {
  if (daughter == null)  {
    return new Pair<>(Boolean.FALSE, Boolean.FALSE);
  }

  FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
  Path rootdir = this.services.getMasterFileSystem().getRootDir();
  Path tabledir = CommonFSUtils.getTableDir(rootdir, daughter.getTable());

  Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName());

  HRegionFileSystem regionFs;

  try {
    if (!CommonFSUtils.isExists(fs, daughterRegionDir)) {
      return new Pair<>(Boolean.FALSE, Boolean.FALSE);
    }
  } catch (IOException ioe) {
    LOG.error("Error trying to determine if daughter region exists, " +
             "assuming exists and has references", ioe);
    return new Pair<>(Boolean.TRUE, Boolean.TRUE);
  }

  boolean references = false;
  TableDescriptor parentDescriptor = getDescriptor(parent.getTable());
  try {
    regionFs = HRegionFileSystem.openRegionFromFileSystem(
        this.services.getConfiguration(), fs, tabledir, daughter, true);

    for (ColumnFamilyDescriptor family: parentDescriptor.getColumnFamilies()) {
      if ((references = regionFs.hasReferences(family.getNameAsString()))) {
        break;
      }
    }
  } catch (IOException e) {
    LOG.error("Error trying to determine referenced files from : " + daughter.getEncodedName()
        + ", to: " + parent.getEncodedName() + " assuming has references", e);
    return new Pair<>(Boolean.TRUE, Boolean.TRUE);
  }
  return new Pair<>(Boolean.TRUE, references);
}
 
Example 30
Source Project: hbase   Source File: SnapshotManifest.java    License: Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
protected void addRegion(final Path tableDir, final RegionInfo regionInfo, RegionVisitor visitor)
    throws IOException {
  boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo);
  try {
    Path baseDir = tableDir;
    // Open the RegionFS
    if (isMobRegion) {
      baseDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), regionInfo.getTable());
    }
    HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, rootFs,
      baseDir, regionInfo, true);
    monitor.rethrowException();

    // 1. dump region meta info into the snapshot directory
    LOG.debug("Storing region-info for snapshot.");
    Object regionData = visitor.regionOpen(regionInfo);
    monitor.rethrowException();

    // 2. iterate through all the stores in the region
    LOG.debug("Creating references for hfiles");

    // This ensures that we have an atomic view of the directory as long as we have < ls limit
    // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files
    // in batches and may miss files being added/deleted. This could be more robust (iteratively
    // checking to see if we have all the files until we are sure), but the limit is currently
    // 1000 files/batch, far more than the number of store files under a single column family.
    Collection<String> familyNames = regionFs.getFamilies();
    if (familyNames != null) {
      for (String familyName: familyNames) {
        Object familyData = visitor.familyOpen(regionData, Bytes.toBytes(familyName));
        monitor.rethrowException();

        Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName);
        if (storeFiles == null) {
          if (LOG.isDebugEnabled()) {
            LOG.debug("No files under family: " + familyName);
          }
          continue;
        }

        // 2.1. build the snapshot reference for the store
        // iterate through all the store's files and create "references".
        addReferenceFiles(visitor, regionData, familyData, storeFiles, false);

        visitor.familyClose(regionData, familyData);
      }
    }
    visitor.regionClose(regionData);
  } catch (IOException e) {
    // the mob directory might not be created yet, so do nothing when it is a mob region
    if (!isMobRegion) {
      throw e;
    }
  }
}