org.apache.hadoop.hbase.regionserver.HRegionFileSystem Java Examples
The following examples show how to use
org.apache.hadoop.hbase.regionserver.HRegionFileSystem.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0 | 6 votes |
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); HColumnDescriptor cfd = new HColumnDescriptor(family); if (ttl > 0) { cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl)); } cfd.setMaxVersions(10); htd.addFamily(cfd); htd.addCoprocessor(TransactionProcessor.class.getName()); Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName()); FileSystem fs = FileSystem.get(conf); assertTrue(fs.mkdirs(tablePath)); WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog"); WAL hLog = walFactory.getWAL(new byte[]{1}, null); HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName)); HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo); return new HRegion(regionFS, hLog, conf, htd, new LocalRegionServerServices(conf, ServerName.valueOf( InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis()))); }
Example #2
Source File: OfflineMetaRebuildTestCore.java From hbase with Apache License 2.0 | 6 votes |
protected RegionInfo createRegion(Configuration conf, final Table htbl, byte[] startKey, byte[] endKey) throws IOException { Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME); RegionInfo hri = RegionInfoBuilder.newBuilder(htbl.getName()) .setStartKey(startKey) .setEndKey(endKey) .build(); LOG.info("manually adding regioninfo and hdfs data: " + hri.toString()); Path rootDir = CommonFSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); Path p = new Path(CommonFSUtils.getTableDir(rootDir, htbl.getName()), hri.getEncodedName()); fs.mkdirs(p); Path riPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE); FSDataOutputStream out = fs.create(riPath); out.write(RegionInfo.toDelimitedByteArray(hri)); out.close(); // add to meta. MetaTableAccessor.addRegionToMeta(TEST_UTIL.getConnection(), hri); meta.close(); return hri; }
Example #3
Source File: TestMajorCompactionRequest.java From hbase with Apache License 2.0 | 6 votes |
private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles, List<StoreFileInfo> storeFiles, long referenceFileTimestamp) throws IOException { FileSystem fileSystem = mock(FileSystem.class); if (hasReferenceFiles) { FileStatus fileStatus = mock(FileStatus.class); doReturn(referenceFileTimestamp).when(fileStatus).getModificationTime(); doReturn(fileStatus).when(fileSystem).getFileLinkStatus(isA(Path.class)); } HRegionFileSystem mockSystem = mock(HRegionFileSystem.class); doReturn(info).when(mockSystem).getRegionInfo(); doReturn(regionStoreDir).when(mockSystem).getStoreDir(FAMILY); doReturn(hasReferenceFiles).when(mockSystem).hasReferences(anyString()); doReturn(storeFiles).when(mockSystem).getStoreFiles(anyString()); doReturn(fileSystem).when(mockSystem).getFileSystem(); return mockSystem; }
Example #4
Source File: TestMajorCompactionRequest.java From hbase with Apache License 2.0 | 6 votes |
@Test public void testIfWeHaveNewReferenceFilesButOldStoreFiles() throws Exception { // this tests that reference files that are new, but have older timestamps for the files // they reference still will get compacted. TableName table = TableName.valueOf("TestMajorCompactor"); TableDescriptor htd = UTILITY.createTableDescriptor(table, Bytes.toBytes(FAMILY)); RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); HRegion region = HBaseTestingUtility.createRegionAndWAL(hri, rootRegionDir, UTILITY.getConfiguration(), htd); Configuration configuration = mock(Configuration.class); // the reference file timestamp is newer List<StoreFileInfo> storeFiles = mockStoreFiles(regionStoreDir, 4, 101); List<Path> paths = storeFiles.stream().map(StoreFileInfo::getPath).collect(Collectors.toList()); // the files that are referenced are older, thus we still compact. HRegionFileSystem fileSystem = mockFileSystem(region.getRegionInfo(), true, storeFiles, 50); MajorCompactionRequest majorCompactionRequest = spy(new MajorCompactionRequest(configuration, region.getRegionInfo(), Sets.newHashSet(FAMILY))); doReturn(mock(Connection.class)).when(majorCompactionRequest).getConnection(eq(configuration)); doReturn(paths).when(majorCompactionRequest).getReferenceFilePaths(any(FileSystem.class), any(Path.class)); doReturn(fileSystem).when(majorCompactionRequest).getFileSystem(any(Connection.class)); Set<String> result = majorCompactionRequest.getStoresRequiringCompaction(Sets.newHashSet("a"), 100); assertEquals(FAMILY, Iterables.getOnlyElement(result)); }
Example #5
Source File: MajorCompactionRequest.java From hbase with Apache License 2.0 | 6 votes |
boolean shouldCFBeCompacted(HRegionFileSystem fileSystem, String family, long ts) throws IOException { // do we have any store files? Collection<StoreFileInfo> storeFiles = fileSystem.getStoreFiles(family); if (storeFiles == null) { LOG.info("Excluding store: " + family + " for compaction for region: " + fileSystem .getRegionInfo().getEncodedName(), " has no store files"); return false; } // check for reference files if (fileSystem.hasReferences(family) && familyHasReferenceFile(fileSystem, family, ts)) { LOG.info("Including store: " + family + " with: " + storeFiles.size() + " files for compaction for region: " + fileSystem.getRegionInfo().getEncodedName()); return true; } // check store file timestamps boolean includeStore = this.shouldIncludeStore(fileSystem, family, storeFiles, ts); if (!includeStore) { LOG.info("Excluding store: " + family + " for compaction for region: " + fileSystem .getRegionInfo().getEncodedName() + " already compacted"); } return includeStore; }
Example #6
Source File: MajorCompactionTTLRequest.java From hbase with Apache License 2.0 | 6 votes |
@Override protected boolean shouldIncludeStore(HRegionFileSystem fileSystem, String family, Collection<StoreFileInfo> storeFiles, long ts) throws IOException { for (StoreFileInfo storeFile : storeFiles) { // Lets only compact when all files are older than TTL if (storeFile.getModificationTime() >= ts) { LOG.info("There is atleast one file in store: " + family + " file: " + storeFile.getPath() + " with timestamp " + storeFile.getModificationTime() + " for region: " + fileSystem.getRegionInfo().getEncodedName() + " older than TTL: " + ts); return false; } } return true; }
Example #7
Source File: RestoreSnapshotHelper.java From hbase with Apache License 2.0 | 6 votes |
/** * @return the set of the regions contained in the table */ private List<RegionInfo> getTableRegions() throws IOException { LOG.debug("get table regions: " + tableDir); FileStatus[] regionDirs = CommonFSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs)); if (regionDirs == null) { return null; } List<RegionInfo> regions = new ArrayList<>(regionDirs.length); for (int i = 0; i < regionDirs.length; ++i) { RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDirs[i].getPath()); regions.add(hri); } LOG.debug("found " + regions.size() + " regions for table=" + tableDesc.getTableName().getNameAsString()); return regions; }
Example #8
Source File: SplitTableRegionProcedure.java From hbase with Apache License 2.0 | 6 votes |
private Pair<Path, Path> splitStoreFile(HRegionFileSystem regionFs, byte[] family, HStoreFile sf) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("pid=" + getProcId() + " splitting started for store file: " + sf.getPath() + " for region: " + getParentRegion().getShortNameToLog()); } final byte[] splitRow = getSplitRow(); final String familyName = Bytes.toString(family); final Path path_first = regionFs.splitStoreFile(this.daughterOneRI, familyName, sf, splitRow, false, splitPolicy); final Path path_second = regionFs.splitStoreFile(this.daughterTwoRI, familyName, sf, splitRow, true, splitPolicy); if (LOG.isDebugEnabled()) { LOG.debug("pid=" + getProcId() + " splitting complete for store file: " + sf.getPath() + " for region: " + getParentRegion().getShortNameToLog()); } return new Pair<Path,Path>(path_first, path_second); }
Example #9
Source File: SplitTableRegionProcedure.java From hbase with Apache License 2.0 | 6 votes |
/** * Create daughter regions */ @VisibleForTesting public void createDaughterRegions(final MasterProcedureEnv env) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tabledir = CommonFSUtils.getTableDir(mfs.getRootDir(), getTableName()); final FileSystem fs = mfs.getFileSystem(); HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, tabledir, getParentRegion(), false); regionFs.createSplitsDir(daughterOneRI, daughterTwoRI); Pair<Integer, Integer> expectedReferences = splitStoreFiles(env, regionFs); assertReferenceFileCount(fs, expectedReferences.getFirst(), regionFs.getSplitsDir(daughterOneRI)); //Move the files from the temporary .splits to the final /table/region directory regionFs.commitDaughterRegion(daughterOneRI); assertReferenceFileCount(fs, expectedReferences.getFirst(), new Path(tabledir, daughterOneRI.getEncodedName())); assertReferenceFileCount(fs, expectedReferences.getSecond(), regionFs.getSplitsDir(daughterTwoRI)); regionFs.commitDaughterRegion(daughterTwoRI); assertReferenceFileCount(fs, expectedReferences.getSecond(), new Path(tabledir, daughterTwoRI.getEncodedName())); }
Example #10
Source File: MergeTableRegionsProcedure.java From hbase with Apache License 2.0 | 6 votes |
/** * Create reference file(s) to parent region hfiles in the <code>mergeDir</code> * @param regionFs merge parent region file system * @param mergeDir the temp directory in which we are accumulating references. */ private void mergeStoreFiles(final MasterProcedureEnv env, final HRegionFileSystem regionFs, final Path mergeDir) throws IOException { final TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(getTableName()); for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) { String family = hcd.getNameAsString(); final Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(family); if (storeFiles != null && storeFiles.size() > 0) { for (StoreFileInfo storeFileInfo : storeFiles) { // Create reference file(s) to parent region file here in mergedDir. // As this procedure is running on master, use CacheConfig.DISABLED means // don't cache any block. regionFs.mergeStoreFile(mergedRegion, family, new HStoreFile( storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED), mergeDir); } } } }
Example #11
Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0 | 6 votes |
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); HColumnDescriptor cfd = new HColumnDescriptor(family); if (ttl > 0) { cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl)); } cfd.setMaxVersions(10); htd.addFamily(cfd); htd.addCoprocessor(TransactionProcessor.class.getName()); Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName()); Path hlogPath = new Path(FSUtils.getRootDir(conf) + "/hlog"); FileSystem fs = FileSystem.get(conf); assertTrue(fs.mkdirs(tablePath)); HLog hLog = HLogFactory.createHLog(fs, hlogPath, tableName, conf); HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName)); HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo); return new HRegion(regionFS, hLog, conf, htd, new MockRegionServerServices(conf, null)); }
Example #12
Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0 | 6 votes |
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); HColumnDescriptor cfd = new HColumnDescriptor(family); if (ttl > 0) { cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl)); } cfd.setMaxVersions(10); htd.addFamily(cfd); htd.addCoprocessor(TransactionProcessor.class.getName()); Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName()); FileSystem fs = FileSystem.get(conf); assertTrue(fs.mkdirs(tablePath)); WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog"); WAL hLog = walFactory.getWAL(new byte[]{1}); HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName)); HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo); return new HRegion(regionFS, hLog, conf, htd, new LocalRegionServerServices(conf, ServerName.valueOf( InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis()))); }
Example #13
Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0 | 6 votes |
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); HColumnDescriptor cfd = new HColumnDescriptor(family); if (ttl > 0) { cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl)); } cfd.setMaxVersions(10); htd.addFamily(cfd); htd.addCoprocessor(TransactionProcessor.class.getName()); Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName()); Path hlogPath = new Path(FSUtils.getRootDir(conf) + "/hlog"); FileSystem fs = FileSystem.get(conf); assertTrue(fs.mkdirs(tablePath)); HLog hLog = HLogFactory.createHLog(fs, hlogPath, tableName, conf); HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName)); HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo); return new HRegion(regionFS, hLog, conf, htd, new MockRegionServerServices(conf, null)); }
Example #14
Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0 | 6 votes |
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); HColumnDescriptor cfd = new HColumnDescriptor(family); if (ttl > 0) { cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl)); } cfd.setMaxVersions(10); htd.addFamily(cfd); htd.addCoprocessor(TransactionProcessor.class.getName()); Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName()); FileSystem fs = FileSystem.get(conf); assertTrue(fs.mkdirs(tablePath)); WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog"); WAL hLog = walFactory.getWAL(new byte[]{1}, null); HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName)); HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo); return new HRegion(regionFS, hLog, conf, htd, new LocalRegionServerServices(conf, ServerName.valueOf( InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis()))); }
Example #15
Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0 | 6 votes |
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); HColumnDescriptor cfd = new HColumnDescriptor(family); if (ttl > 0) { cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl)); } cfd.setMaxVersions(10); htd.addFamily(cfd); htd.addCoprocessor(TransactionProcessor.class.getName()); Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName()); FileSystem fs = FileSystem.get(conf); assertTrue(fs.mkdirs(tablePath)); WALFactory walFactory = new WALFactory(conf, tableName + ".hlog"); HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); WAL hLog = walFactory.getWAL(info); HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName)); HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo); return new HRegion(regionFS, hLog, conf, htd, new LocalRegionServerServices(conf, ServerName.valueOf( InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis()))); }
Example #16
Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0 | 6 votes |
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); HColumnDescriptor cfd = new HColumnDescriptor(family); if (ttl > 0) { cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl)); } cfd.setMaxVersions(10); htd.addFamily(cfd); htd.addCoprocessor(TransactionProcessor.class.getName()); Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName()); FileSystem fs = FileSystem.get(conf); assertTrue(fs.mkdirs(tablePath)); WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog"); WAL hLog = walFactory.getWAL(new byte[]{1}); HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName)); HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo); return new HRegion(regionFS, hLog, conf, htd, new LocalRegionServerServices(conf, ServerName.valueOf( InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis()))); }
Example #17
Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0 | 6 votes |
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); HColumnDescriptor cfd = new HColumnDescriptor(family); if (ttl > 0) { cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl)); } cfd.setMaxVersions(10); htd.addFamily(cfd); htd.addCoprocessor(TransactionProcessor.class.getName()); Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName()); FileSystem fs = FileSystem.get(conf); assertTrue(fs.mkdirs(tablePath)); WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog"); WAL hLog = walFactory.getWAL(new byte[]{1}); HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName)); HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo); return new HRegion(regionFS, hLog, conf, htd, new LocalRegionServerServices(conf, ServerName.valueOf( InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis()))); }
Example #18
Source File: CatalogJanitor.java From hbase with Apache License 2.0 | 5 votes |
/** * If merged region no longer holds reference to the merge regions, archive * merge region on hdfs and perform deleting references in hbase:meta * @return true if we delete references in merged region on hbase:meta and archive * the files on the file system */ private boolean cleanMergeRegion(final RegionInfo mergedRegion, List<RegionInfo> parents) throws IOException { FileSystem fs = this.services.getMasterFileSystem().getFileSystem(); Path rootdir = this.services.getMasterFileSystem().getRootDir(); Path tabledir = CommonFSUtils.getTableDir(rootdir, mergedRegion.getTable()); TableDescriptor htd = getDescriptor(mergedRegion.getTable()); HRegionFileSystem regionFs = null; try { regionFs = HRegionFileSystem.openRegionFromFileSystem( this.services.getConfiguration(), fs, tabledir, mergedRegion, true); } catch (IOException e) { LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName()); } if (regionFs == null || !regionFs.hasReferences(htd)) { LOG.debug("Deleting parents ({}) from fs; merged child {} no longer holds references", parents.stream().map(r -> RegionInfo.getShortNameToLog(r)). collect(Collectors.joining(", ")), mergedRegion); ProcedureExecutor<MasterProcedureEnv> pe = this.services.getMasterProcedureExecutor(); pe.submitProcedure(new GCMultipleMergedRegionsProcedure(pe.getEnvironment(), mergedRegion, parents)); for (RegionInfo ri: parents) { // The above scheduled GCMultipleMergedRegionsProcedure does the below. // Do we need this? this.services.getAssignmentManager().getRegionStates().deleteRegion(ri); this.services.getServerManager().removeRegion(ri); } return true; } return false; }
Example #19
Source File: TestMajorCompactionTTLRequest.java From hbase with Apache License 2.0 | 5 votes |
private MajorCompactionTTLRequest makeMockRequest(List<StoreFileInfo> storeFiles) throws IOException { Configuration configuration = mock(Configuration.class); RegionInfo regionInfo = mock(RegionInfo.class); when(regionInfo.getEncodedName()).thenReturn("HBase"); when(regionInfo.getTable()).thenReturn(TableName.valueOf("foo")); MajorCompactionTTLRequest request = new MajorCompactionTTLRequest(configuration, regionInfo); MajorCompactionTTLRequest spy = spy(request); HRegionFileSystem fileSystem = mockFileSystem(regionInfo, false, storeFiles); doReturn(fileSystem).when(spy).getFileSystem(isA(Connection.class)); doReturn(mock(Connection.class)).when(spy).getConnection(eq(configuration)); return spy; }
Example #20
Source File: TestMajorCompactionRequest.java From hbase with Apache License 2.0 | 5 votes |
private MajorCompactionRequest makeMockRequest(List<StoreFileInfo> storeFiles, boolean references) throws IOException { Configuration configuration = mock(Configuration.class); RegionInfo regionInfo = mock(RegionInfo.class); when(regionInfo.getEncodedName()).thenReturn("HBase"); when(regionInfo.getTable()).thenReturn(TableName.valueOf("foo")); MajorCompactionRequest request = new MajorCompactionRequest(configuration, regionInfo, Sets.newHashSet("a")); MajorCompactionRequest spy = spy(request); HRegionFileSystem fileSystem = mockFileSystem(regionInfo, references, storeFiles); doReturn(fileSystem).when(spy).getFileSystem(isA(Connection.class)); doReturn(mock(Connection.class)).when(spy).getConnection(eq(configuration)); return spy; }
Example #21
Source File: SnapshotTestingUtils.java From hbase with Apache License 2.0 | 5 votes |
private RegionData[] createTable(final TableDescriptor htd, final int nregions) throws IOException { Path tableDir = CommonFSUtils.getTableDir(rootDir, htd.getTableName()); new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false); assertTrue(nregions % 2 == 0); RegionData[] regions = new RegionData[nregions]; for (int i = 0; i < regions.length; i += 2) { byte[] startKey = Bytes.toBytes(0 + i * 2); byte[] endKey = Bytes.toBytes(1 + i * 2); // First region, simple with one plain hfile. RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()) .setStartKey(startKey) .setEndKey(endKey) .build(); HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri); regions[i] = new RegionData(tableDir, hri, 3); for (int j = 0; j < regions[i].files.length; ++j) { Path storeFile = createStoreFile(rfs.createTempName()); regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile); } // Second region, used to test the split case. // This region contains a reference to the hfile in the first region. startKey = Bytes.toBytes(2 + i * 2); endKey = Bytes.toBytes(3 + i * 2); hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri); regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length); for (int j = 0; j < regions[i].files.length; ++j) { String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName(); Path refFile = createStoreFile(new Path(rootDir, refName)); regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile); } } return regions; }
Example #22
Source File: MajorCompactionRequest.java From hbase with Apache License 2.0 | 5 votes |
@VisibleForTesting HRegionFileSystem getFileSystem(Connection connection) throws IOException { Admin admin = connection.getAdmin(); return HRegionFileSystem.openRegionFromFileSystem(admin.getConfiguration(), CommonFSUtils.getCurrentFileSystem(admin.getConfiguration()), CommonFSUtils.getTableDir( CommonFSUtils.getRootDir(admin.getConfiguration()), region.getTable()), region, true); }
Example #23
Source File: MajorCompactionRequest.java From hbase with Apache License 2.0 | 5 votes |
protected boolean familyHasReferenceFile(HRegionFileSystem fileSystem, String family, long ts) throws IOException { List<Path> referenceFiles = getReferenceFilePaths(fileSystem.getFileSystem(), fileSystem.getStoreDir(family)); for (Path referenceFile : referenceFiles) { FileStatus status = fileSystem.getFileSystem().getFileLinkStatus(referenceFile); if (status.getModificationTime() < ts) { LOG.info("Including store: " + family + " for compaction for region: " + fileSystem .getRegionInfo().getEncodedName() + " (reference store files)"); return true; } } return false; }
Example #24
Source File: MajorCompactionRequest.java From hbase with Apache License 2.0 | 5 votes |
protected boolean shouldIncludeStore(HRegionFileSystem fileSystem, String family, Collection<StoreFileInfo> storeFiles, long ts) throws IOException { for (StoreFileInfo storeFile : storeFiles) { if (storeFile.getModificationTime() < ts) { LOG.info("Including store: " + family + " with: " + storeFiles.size() + " files for compaction for region: " + fileSystem.getRegionInfo().getEncodedName()); return true; } } return false; }
Example #25
Source File: MajorCompactionRequest.java From hbase with Apache License 2.0 | 5 votes |
Set<String> getStoresRequiringCompaction(Set<String> requestedStores, long timestamp) throws IOException { try(Connection connection = getConnection(configuration)) { HRegionFileSystem fileSystem = getFileSystem(connection); Set<String> familiesToCompact = Sets.newHashSet(); for (String family : requestedStores) { if (shouldCFBeCompacted(fileSystem, family, timestamp)) { familiesToCompact.add(family); } } return familiesToCompact; } }
Example #26
Source File: MajorCompactionTTLRequest.java From hbase with Apache License 2.0 | 5 votes |
Map<String, Long> getStoresRequiringCompaction(TableDescriptor htd) throws IOException { try(Connection connection = getConnection(configuration)) { HRegionFileSystem fileSystem = getFileSystem(connection); Map<String, Long> familyTTLMap = Maps.newHashMap(); for (ColumnFamilyDescriptor descriptor : htd.getColumnFamilies()) { long ts = getColFamilyCutoffTime(descriptor); // If the table's TTL is forever, lets not compact any of the regions. if (ts > 0 && shouldCFBeCompacted(fileSystem, descriptor.getNameAsString(), ts)) { familyTTLMap.put(descriptor.getNameAsString(), ts); } } return familyTTLMap; } }
Example #27
Source File: MergeTableRegionsProcedure.java From hbase with Apache License 2.0 | 5 votes |
/** * Clean up a merged region on rollback after failure. */ private void cleanupMergedRegion(final MasterProcedureEnv env) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); TableName tn = this.regionsToMerge[0].getTable(); final Path tabledir = CommonFSUtils.getTableDir(mfs.getRootDir(), tn); final FileSystem fs = mfs.getFileSystem(); // See createMergedRegion above where we specify the merge dir as being in the // FIRST merge parent region. HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, tabledir, regionsToMerge[0], false); regionFs.cleanupMergedRegion(mergedRegion); }
Example #28
Source File: SnapshotManifestV1.java From hbase with Apache License 2.0 | 4 votes |
static SnapshotRegionManifest buildManifestFromDisk(final Configuration conf, final FileSystem fs, final Path tableDir, final RegionInfo regionInfo) throws IOException { HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs, tableDir, regionInfo, true); SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder(); // 1. dump region meta info into the snapshot directory LOG.debug("Storing region-info for snapshot."); manifest.setRegionInfo(ProtobufUtil.toRegionInfo(regionInfo)); // 2. iterate through all the stores in the region LOG.debug("Creating references for hfiles"); // This ensures that we have an atomic view of the directory as long as we have < ls limit // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in // batches and may miss files being added/deleted. This could be more robust (iteratively // checking to see if we have all the files until we are sure), but the limit is currently 1000 // files/batch, far more than the number of store files under a single column family. Collection<String> familyNames = regionFs.getFamilies(); if (familyNames != null) { for (String familyName: familyNames) { Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName, false); if (storeFiles == null) { LOG.debug("No files under family: " + familyName); continue; } // 2.1. build the snapshot reference for the store SnapshotRegionManifest.FamilyFiles.Builder family = SnapshotRegionManifest.FamilyFiles.newBuilder(); family.setFamilyName(UnsafeByteOperations.unsafeWrap(Bytes.toBytes(familyName))); if (LOG.isDebugEnabled()) { LOG.debug("Adding snapshot references for " + storeFiles + " hfiles"); } // 2.2. iterate through all the store's files and create "references". int i = 0; int sz = storeFiles.size(); for (StoreFileInfo storeFile: storeFiles) { // create "reference" to this store file. LOG.debug("Adding reference for file ("+ (++i) +"/" + sz + "): " + storeFile.getPath()); SnapshotRegionManifest.StoreFile.Builder sfManifest = SnapshotRegionManifest.StoreFile.newBuilder(); sfManifest.setName(storeFile.getPath().getName()); family.addStoreFiles(sfManifest.build()); } manifest.addFamilyFiles(family.build()); } } return manifest.build(); }
Example #29
Source File: SnapshotManifestV1.java From hbase with Apache License 2.0 | 4 votes |
@Override public void familyClose(final HRegionFileSystem region, final Path family) { }
Example #30
Source File: SnapshotManifestV1.java From hbase with Apache License 2.0 | 4 votes |
@Override public Path familyOpen(final HRegionFileSystem snapshotRegionFs, final byte[] familyName) { Path familyDir = snapshotRegionFs.getStoreDir(Bytes.toString(familyName)); return familyDir; }